x64/InterruptManager.cc

/*
 * Copyright (c) 2008-2014, Pedigree Developers
 *
 * Please see the CONTRIB file in the root of the source tree for a full
 * list of contributors.
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */
 
#include "InterruptManager.h"
#include "pedigree/kernel/ActivityDiagnostics.h"
#include "pedigree/kernel/LockGuard.h"
#include "pedigree/kernel/Log.h"
#include "pedigree/kernel/Metrics.h"
#include "pedigree/kernel/linker/KernelElf.h"
#include "pedigree/kernel/process/Thread.h"
#include "pedigree/kernel/processor/InterruptHandler.h"
#include "pedigree/kernel/processor/Processor.h"
#include "pedigree/kernel/processor/ProcessorInformation.h"
#include "pedigree/kernel/processor/state.h"
#include "pedigree/kernel/utilities/StaticString.h"
 
#if DEBUGGER
#include "pedigree/kernel/debugger/Debugger.h"
#endif
 
#if THREADS
#include "pedigree/kernel/Subsystem.h"
#include "pedigree/kernel/process/InterruptTimeAccounting.h"
#include "pedigree/kernel/process/PerProcessorScheduler.h"
#include "pedigree/kernel/process/Process.h"
#endif
 
static_assert(sizeof(X64InterruptState) == 208, "IST frame copy must match the saved frame");
 
#if PEDIGREE_X64_USER_ENTRY_DIAGNOSTICS
struct alignas(64) NmiEntryDiagnostic {
  uint64_t count;
  uint64_t rip;
  uint64_t cs;
  uint64_t rsp;
  uint64_t information;
  uint64_t index;
  uint64_t activeGs;
  uint64_t frame;
};
static_assert(sizeof(NmiEntryDiagnostic) == 64);
extern "C" {
SYMBOL_HIDDEN volatile NmiEntryDiagnostic pedigree_nmi_entry_diagnostics[256] = {};
}
#endif
 
static const char* g_ExceptionNames[] = {"Divide Error",
                                         "Debug",
                                         "NMI Interrupt",
                                         "Breakpoint",
                                         "Overflow",
                                         "BOUND Range Exceeded",
                                         "Invalid Opcode",
                                         "Device Not Available",
                                         "Double Fault",
                                         "Coprocessor Segment Overrun", /* recent IA-32 processors
                                                                           don't generate this */
                                         "Invalid TSS",
                                         "Segment Not Present",
                                         "Stack Fault",
                                         "General Protection Fault",
                                         "Page Fault",
                                         "FPU Floating-Point Error",
                                         "Alignment Check",
                                         "Machine-Check",
                                         "SIMD Floating-Point Exception",
                                         "Reserved: Interrupt 19",
                                         "Reserved: Interrupt 20",
                                         "Reserved: Interrupt 21",
                                         "Reserved: Interrupt 22",
                                         "Reserved: Interrupt 23",
                                         "Reserved: Interrupt 24",
                                         "Reserved: Interrupt 25",
                                         "Reserved: Interrupt 26",
                                         "Reserved: Interrupt 27",
                                         "Reserved: Interrupt 28",
                                         "Reserved: Interrupt 29",
                                         "Reserved: Interrupt 30",
                                         "Reserved: Interrupt 31"};
 
X64InterruptManager X64InterruptManager::m_Instance;
 
InterruptManager& InterruptManager::instance() {
  return X64InterruptManager::instance();
}
 
bool X64InterruptManager::registerInterruptHandler(size_t nInterruptNumber,
                                                   InterruptHandler* pHandler) {
  // Lock the class until the end of the function
  LockGuard<Spinlock> lock(m_Lock);
 
  // Sanity checks
  if (UNLIKELY(nInterruptNumber >= 256))
    return false;
  InterruptHandler* current = __atomic_load_n(&m_pHandler[nInterruptNumber], __ATOMIC_ACQUIRE);
  if (UNLIKELY(pHandler != 0 && current != 0))
    return false;
  if (UNLIKELY(pHandler == 0 && current == 0))
    return false;
 
  // Exceptions and IRQs can re-enter on this CPU while this lock is held.
  // Publish the complete old or new pointer without involving that lock.
  return __atomic_compare_exchange_n(&m_pHandler[nInterruptNumber], &current, pHandler, false,
                                     __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE);
}
 
#if DEBUGGER
 
bool X64InterruptManager::registerInterruptHandlerDebugger(size_t nInterruptNumber,
                                                           InterruptHandler* pHandler) {
  // Lock the class until the end of the function
  LockGuard<Spinlock> lock(m_Lock);
 
  // Sanity checks
  if (UNLIKELY(nInterruptNumber >= 256))
    return false;
  InterruptHandler* current = __atomic_load_n(&m_pDbgHandler[nInterruptNumber], __ATOMIC_ACQUIRE);
  if (UNLIKELY(pHandler != 0 && current != 0))
    return false;
  if (UNLIKELY(pHandler == 0 && current == 0))
    return false;
 
  return __atomic_compare_exchange_n(&m_pDbgHandler[nInterruptNumber], &current, pHandler, false,
                                     __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE);
}
size_t X64InterruptManager::getBreakpointInterruptNumber() {
  return 3;
}
size_t X64InterruptManager::getDebugInterruptNumber() {
  return 1;
}
 
#endif
 
void X64InterruptManager::interrupt(InterruptState& interruptState) {
#if PEDIGREE_X64_USER_ENTRY_DIAGNOSTICS
  if (interruptState.getInterruptNumber() == 2) {
    uintptr_t information;
    size_t index;
    asm volatile("movq %%gs:16, %0; movq %%gs:24, %1"
                 : "=r"(information), "=r"(index)
                 :
                 : "memory");
    uint32_t low, high;
    asm volatile("rdmsr" : "=a"(low), "=d"(high) : "c"(0xc0000101));
    if (index < 256) {
      auto& record = pedigree_nmi_entry_diagnostics[index];
      record.rip = interruptState.m_Rip;
      record.cs = interruptState.m_Cs;
      record.rsp = interruptState.m_Rsp;
      record.information = information;
      record.index = index;
      record.activeGs = (static_cast<uint64_t>(high) << 32) | low;
      record.frame = reinterpret_cast<uintptr_t>(&interruptState);
      __atomic_store_n(&record.count, record.count + 1, __ATOMIC_RELEASE);
    }
    return;
  }
#endif
  InterruptTimeAccounting accounting(!interruptState.kernelMode());
  size_t nIntNumber = interruptState.getInterruptNumber();
  Metrics::increment(nIntNumber < 32 ? Metrics::Counter::Exception : Metrics::Counter::Interrupt);
  ActivityDiagnostics::InterruptScope activityScope(nIntNumber);
 
#if DEBUGGER
  {
    InterruptHandler* pHandler =
        __atomic_load_n(&m_Instance.m_pDbgHandler[nIntNumber], __ATOMIC_ACQUIRE);
 
    // Call the kernel debugger's handler, if any
    if (pHandler != 0) {
      ExecutionContextGuard debuggerContext(ExecutionContext::DebuggerTrap);
      pHandler->interrupt(nIntNumber, interruptState);
    }
  }
#endif
 
  InterruptHandler* pHandler =
      __atomic_load_n(&m_Instance.m_pHandler[nIntNumber], __ATOMIC_ACQUIRE);
 
  // Call the normal interrupt handler, if any
  if (LIKELY(pHandler != 0)) {
    pHandler->interrupt(nIntNumber, interruptState);
    return;
  }
 
// Were we running in the kernel, or user space?
// User space processes have a subsystem, kernel ones do not.
#if THREADS
  Thread* pThread = Processor::information().getCurrentThread();
  if (pThread) {
    Process* pProcess = pThread->getParent();
    if (pProcess) {
      Subsystem* pSubsystem = pProcess->getSubsystem();
      if (pSubsystem && !interruptState.kernelMode()) {
        size_t exceptionType = static_cast<size_t>(Subsystem::Other);
        if (UNLIKELY(nIntNumber == 0)) {
          exceptionType = static_cast<size_t>(Subsystem::DivideByZero);
        } else if (UNLIKELY(nIntNumber == 6)) {
          exceptionType = static_cast<size_t>(Subsystem::InvalidOpcode);
        } else if (UNLIKELY(nIntNumber == 13)) {
          exceptionType = static_cast<size_t>(Subsystem::GeneralProtectionFault);
        } else if (UNLIKELY(nIntNumber == 16)) {
          exceptionType = static_cast<size_t>(Subsystem::FpuError);
        } else if (UNLIKELY(nIntNumber == 19)) {
          exceptionType = static_cast<size_t>(Subsystem::SpecialFpuError);
        }
 
        if (exceptionType != static_cast<size_t>(Subsystem::Other)) {
          pThread->deferSubsystemException(exceptionType, 0, interruptState.getErrorCode());
          return;
        }
      }
    }
  }
#endif
 
  // unhandled interrupt, check for an exception (interrupts 0-31 inclusive
  // are reserved, not for use by system programmers)
  if (LIKELY(nIntNumber < 32 && nIntNumber != 1 && nIntNumber != 3)) {
    // TODO:: Check for debugger initialisation.
    // TODO: register dump, maybe a breakpoint so the deubbger can take
    // over?
    // TODO: Rework this
    // for now just print out the exception name and number
    LargeStaticString e;
    e.append("Exception #0x");
    e.append(nIntNumber, 16);
    e.append(": \"");
    e.append(g_ExceptionNames[nIntNumber]);
    e.append("\"");
 
    // Shutdown may have retired both log callbacks and debugger services.
    if (interruptState.kernelMode() && KernelElf::instance().isShuttingDown()) {
      e.append(" RIP 0x");
      e.append(interruptState.getInstructionPointer(), 16, 16, '0');
      e.append(", error code 0x");
      e.append(interruptState.getErrorCode(), 16, 8, '0');
      panic(e);
    }
 
#if THREADS
    e.append(" CPU=");
    e.append(Processor::id());
    if (pThread) {
      Process* pParent = pThread->getParent();
      if (pParent) {
        e.append(" PID=");
        e.append(pParent->getId());
      }
      e.append(" TID=");
      e.append(pThread->getId());
    }
#endif
 
    if (nIntNumber == 14) {
      uint64_t cr2;
      asm volatile("mov %%cr2, %%rax" : "=a"(cr2));
      e.append(" at 0x");
      e.append(cr2, 16, 16, '0');
      e.append(", errorcode 0x");
      e.append(interruptState.m_Errorcode, 16, 8, '0');
    }
 
    if (nIntNumber == 13) {
      // GPF
      if (interruptState.m_Errorcode) {
        e.append(" errorcode 0x");
        e.append(interruptState.m_Errorcode, 16, 8, '0');
      }
      e.append(" RIP 0x");
      e.append(interruptState.getInstructionPointer(), 16, 16, '0');
    }
 
    if (nIntNumber == 8) {
      // On amd64, we actually have a functional InterruptState.
      ERROR_NOLOCK("(double fault, system is very unhappy)");
 
      uint64_t cr2;
      asm volatile("mov %%cr2, %%rax" : "=a"(cr2));
      NOTICE_NOLOCK("  -> #DF possibly caused by #PF at " << Hex << cr2 << ".");
    }
 
    // Write the failure into the kernel log before launching the debugger.
    ERROR(static_cast<const char*>(e));
 
#if DEBUGGER
    Debugger::instance().start(interruptState, e);
#else
    panic(e);
#endif
  }
}
 
void X64InterruptManager::returnFromInterrupt(InterruptState& interruptState) {
  const size_t vector = interruptState.getInterruptNumber();
#if PEDIGREE_X64_USER_ENTRY_DIAGNOSTICS
  if (vector == 2)
    return;
#endif
#if THREADS
  if (vector >= 32 && Processor::information().getCurrentThread()) {
    Processor::information().getScheduler().servicePendingScheduling();
  }
#endif
  if (interruptState.kernelMode()) {
    return;
  }
 
  Thread* thread = Processor::information().getCurrentThread();
  if (!thread) {
    return;
  }
 
  if (vector == 2 || vector == 8 || vector == 18) {
    // NMI blocking remains active until IRET, and #DF uses an IST stack.
    // #MC likewise is not a boundary on which arbitrary thread work can
    // safely suspend the architectural exception return.
    Processor::setInterrupts(false);
    InterruptTimeAccounting::finishUserReturn(thread);
    return;
  }
 
  PerProcessorScheduler& scheduler = Processor::information().getScheduler();
  const bool diagnosticSample = scheduler.sampleUserReturnDiagnostics();
  const uint64_t tailStart = diagnosticSample ? ActivityDiagnostics::timestamp() : 0;
 
  // interrupt() has returned, so InterruptTimeAccounting and every raw
  // handler scope are complete. Finish the architecture accounting tail
  // before a terminal transition consumes this root stack.
  Processor::setInterrupts(true);
  bool terminal = false;
  while (true) {
    terminal = scheduler.serviceUserReturnWork(interruptState, UserReturnFrame::Origin::Interrupt,
                                               diagnosticSample);
    if (terminal)
      break;
    bool waited = false;
    const uint64_t affinityStart = diagnosticSample ? ActivityDiagnostics::timestamp() : 0;
    terminal = thread->completeAffinityAtSafePoint(&waited) == AffinityResult::Terminal;
    if (diagnosticSample) {
      ActivityDiagnostics::recordUserReturnStage(
          ActivityDiagnostics::UserReturnStage::InterruptAffinity,
          ActivityDiagnostics::timestamp() - affinityStart);
      if (waited)
        ActivityDiagnostics::recordUserReturnAffinityWait(false);
    }
    if (terminal || !waited)
      break;
    Processor::setInterrupts(true);
  }
  Processor::setInterrupts(false);
  const uint64_t accountingStart = diagnosticSample ? ActivityDiagnostics::timestamp() : 0;
  InterruptTimeAccounting::finishUserReturn(thread);
  if (diagnosticSample) {
    ActivityDiagnostics::recordUserReturnStage(
        ActivityDiagnostics::UserReturnStage::InterruptAccounting,
        ActivityDiagnostics::timestamp() - accountingStart);
    ActivityDiagnostics::recordUserReturnStage(ActivityDiagnostics::UserReturnStage::InterruptTail,
                                               ActivityDiagnostics::timestamp() - tailStart);
  }
  if (terminal) {
    Processor::setInterrupts(true);
    Processor::information().getScheduler().commitUserReturnTerminalState();
    FATAL_NOLOCK("Terminal user-return commit unexpectedly returned");
  }
}
 
//
// Functions only usable in the kernel initialisation phase
//
 
void X64InterruptManager::initialiseProcessor() {
  // BSP/AP bootstrap precedes the permanent GDT and LTR.
  struct {
    uint16_t size;
    uint64_t idt;
  } PACKED idtr = {4095, reinterpret_cast<uintptr_t>(&m_Instance.m_BootstrapIDT)};
 
  asm volatile("lidt %0" ::"m"(idtr) : "memory");
}
 
void X64InterruptManager::initialiseProcessorIst() {
  struct {
    uint16_t size;
    uint64_t idt;
  } PACKED idtr = {4095, reinterpret_cast<uintptr_t>(&m_Instance.m_IDT)};
 
  asm volatile("lidt %0" ::"m"(idtr) : "memory");
}
 
void X64InterruptManager::setInterruptGate(size_t nInterruptNumber, uintptr_t interruptHandler) {
  m_IDT[nInterruptNumber].offset0 = interruptHandler & 0xFFFF;
  m_IDT[nInterruptNumber].selector = 0x08;
  m_IDT[nInterruptNumber].ist = 0;
  m_IDT[nInterruptNumber].flags = 0xEE /*0x8E*/;
  m_IDT[nInterruptNumber].offset1 = (interruptHandler >> 16) & 0xFFFF;
  m_IDT[nInterruptNumber].offset2 = (interruptHandler >> 32) & 0xFFFFFFFF;
  m_IDT[nInterruptNumber].res = 0;
}
 
void X64InterruptManager::setIst(size_t nInterruptNumber, size_t ist) {
  if (ist > 7)
    return;
  m_IDT[nInterruptNumber].ist = ist;
}
 
X64InterruptManager::X64InterruptManager() : m_Lock() {
  // Initialise the pointers to the pHandler
  for (size_t i = 0; i < 256; i++) {
    m_pHandler[i] = 0;
#if DEBUGGER
    m_pDbgHandler[i] = 0;
#endif
  }
 
  // Initialise the IDT
  extern uintptr_t interrupt_handler_array[];
  for (size_t i = 0; i < 256; i++)
    setInterruptGate(i, interrupt_handler_array[i]);
 
  for (size_t i = 0; i < 256; i++)
    m_BootstrapIDT[i] = m_IDT[i];
 
  // Separate stacks keep asynchronous entry and faults in user-return windows
  // away from an interrupted user RSP. CPL3 #DB/#SS/#GP frames move to rsp0.
  setIst(8, 1);
  setIst(2, 2);
  setIst(1, 3);
  setIst(18, 4);
  setIst(13, 5);
  setIst(12, 6);
}
X64InterruptManager::~X64InterruptManager() {}