PerProcessorScheduler.cc

/*
 * Copyright (c) 2008-2014, Pedigree Developers
 *
 * Please see the CONTRIB file in the root of the source tree for a full
 * list of contributors.
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */
 
#include "pedigree/kernel/ActivityDiagnostics.h"
#include "pedigree/kernel/Atomic.h"
#include "pedigree/kernel/LockGuard.h"
#include "pedigree/kernel/Log.h"
#include "pedigree/kernel/Metrics.h"
#include "pedigree/kernel/Spinlock.h"
#include "pedigree/kernel/Subsystem.h"
#include "pedigree/kernel/debugger/commands/LocksCommand.h"
#include "pedigree/kernel/machine/Machine.h"
#include "pedigree/kernel/machine/SchedulerTimer.h"
#include "pedigree/kernel/machine/Timer.h"
#include "pedigree/kernel/machine/Trace.h"
#include "pedigree/kernel/panic.h"
#include "pedigree/kernel/process/Event.h"
#include "pedigree/kernel/process/PerProcessorScheduler.h"
#include "pedigree/kernel/process/Process.h"
#include "pedigree/kernel/process/RoundRobin.h"
#include "pedigree/kernel/process/Scheduler.h"
#include "pedigree/kernel/process/SchedulingAlgorithm.h"
#include "pedigree/kernel/process/TerminationDeferral.h"
#include "pedigree/kernel/process/Thread.h"
#include "pedigree/kernel/process/eventNumbers.h"
#include "pedigree/kernel/processor/PhysicalMemoryManager.h"
#include "pedigree/kernel/processor/Processor.h"
#include "pedigree/kernel/processor/ProcessorInformation.h"
#include "pedigree/kernel/processor/VirtualAddressSpace.h"
#include "pedigree/kernel/processor/state.h"
#include "pedigree/kernel/time/Time.h"
#include "pedigree/kernel/utilities/utility.h"
#if HOSTED
#include "pedigree/kernel/processor/hosted/Processor.h"
#endif
#if X86_COMMON && MULTIPROCESSOR
#include <machine/mach_pc/LocalApic.h>
#include <machine/mach_pc/Pc.h>
#endif
#if PEDIGREE_HOSTED_FUNCTION_PROFILE
#include "pedigree/kernel/processor/hosted/FunctionProfile.h"
#endif
 
static constexpr bool VerboseScheduler = false;
 
namespace {
Atomic<size_t> g_StackDiscardCount(0);
Atomic<size_t> g_EmergencyProcessKillDiscardCount(0);
Atomic<size_t> g_HostedRegressionDiscardCount(0);
Atomic<size_t> g_LegacyAbiDiscardCount(0);
}  // namespace
 
PerProcessorScheduler::PerProcessorScheduler()
    : m_pSchedulingAlgorithm(0),
      m_NewThreadDataLock(),
      m_NewThreadDataCondition(),
      m_NewThreadData(),
      m_DelayedNewThreadData(),
      m_NewThreadAdmissionOpen(false),
      m_StopNewThreadWorker(false),
      m_NewThreadWorker(),
      m_TimeAccountingState(),
      m_DeferredThreadReapStub(),
      m_DeferredThreadReaps(m_DeferredThreadReapStub),
      m_nDeferredThreadReaps(0),
      m_DeferredThreadReapPublicationState(DeferredReapPublicationClosed),
      m_StopTimeAccountingWorker(0),
      m_TimeAccountingWorker(),
      m_TimeAccountingWorkerWaiters(),
      m_TimeAccountingWorkerWake(),
      m_IrqWorkDoorbell(0),
      m_ReschedulePending(0),
      m_RemotePromptPending(0),
      m_ClockDeadline(0),
      m_IrqWorkLock(),
#if HOSTED && PEDIGREE_HOSTED_SMOKE_TESTS
      m_nDeferredThreadReapCompletions(0),
#endif
      m_pIdleThread(0) {
}
 
PerProcessorScheduler::~PerProcessorScheduler() {
  // The add worker can retire a never-started detached Thread while draining.
  // Keep ordinary destruction admission open until that producer is joined.
  stopNewThreadWorker();
 
  SchedulerTimer* pTimer = Machine::instance().getSchedulerTimer();
  if (!pTimer) {
    panic("No scheduler timer present.");
  }
  if (!pTimer->removeHandler(this)) {
    FATAL("Per-processor scheduler lost timer-handler ownership.");
  }
 
  // With timer-driven scheduling quiesced, close publication, drain accepted
  // targets, and finally join the ordinary destruction worker.
  stopTimeAccountingWorker();
}
 
void PerProcessorScheduler::startTimeAccountingWorker(Process* pParent) {
  if (m_TimeAccountingWorker) {
    FATAL("Per-processor time accounting worker started twice.");
  }
 
  m_StopTimeAccountingWorker = 0;
  Thread* worker = new Thread(pParent, timeAccountingWorkerEntry, this, nullptr, false, true, true);
  worker->setName("deferred process time accounting");
  registerWorkerWake(m_TimeAccountingWorkerWake, m_TimeAccountingWorkerWaiters);
  m_TimeAccountingWorker.adopt(worker);
  if (!worker->start()) {
    FATAL("Time accounting worker could not be started.");
  }
 
  while (!m_DeferredThreadReapPublicationState.compareAndSwap(DeferredReapPublicationClosed, 0)) {
    Scheduler::instance().yield();
  }
  {
    LockGuard<Spinlock> guard(m_AffinityQueueLock);
    m_AffinityAdmissionOpen = true;
  }
}
 
void PerProcessorScheduler::stopTimeAccountingWorker() {
  if (!m_TimeAccountingWorker) {
    return;
  }
  {
    LockGuard<Spinlock> guard(m_AffinityQueueLock);
    m_AffinityAdmissionOpen = false;
  }
 
  // Close allocation-free producers before allowing the worker to exit. The
  // low bits cover a publisher which observed open admission just before this
  // transition.
  m_DeferredThreadReapPublicationState |= DeferredReapPublicationClosed;
  while (m_DeferredThreadReapPublicationState.value() & DeferredReapPublicationCountMask) {
    Scheduler::instance().yield();
  }
 
  m_StopTimeAccountingWorker = 1;
  ringIrqWorkDoorbell(m_TimeAccountingWorkerWake);
  serviceIrqWorkDoorbell();
  m_TimeAccountingWorker.join();
  unregisterWorkerWake(m_TimeAccountingWorkerWake);
  if (m_nDeferredThreadReaps.value() || m_AffinityRequests.value()) {
    FATAL("Deferred Thread reap worker stopped with pending targets.");
  }
}
 
int PerProcessorScheduler::timeAccountingWorkerEntry(void* instance) {
  return reinterpret_cast<PerProcessorScheduler*>(instance)->runTimeAccountingWorker();
}
 
int PerProcessorScheduler::runTimeAccountingWorker() {
  TerminationDeferral workerLifetime;
  while (true) {
    const bool stopping = m_StopTimeAccountingWorker.value() != 0;
    const bool pending = m_TimeAccountingState.ready() || m_nDeferredThreadReaps.value() ||
                         m_AffinityRequests.value();
    if (stopping && m_TimeAccountingState.caughtUp() && !m_nDeferredThreadReaps.value() &&
        !m_AffinityRequests.value()) {
      break;
    }
    if (!pending) {
      auto guard = m_TimeAccountingWorkerWaiters.acquire();
      const bool stillPending = m_TimeAccountingState.ready() || m_nDeferredThreadReaps.value() ||
                                m_AffinityRequests.value() || m_StopTimeAccountingWorker.value();
      if (!stillPending) {
        const WaitQueue::WakeReason reason =
            guard.wait(WaitQueue::Channel(), Thread::CondWait, reinterpret_cast<uintptr_t>(this));
        (void)reason;
      }
      continue;
    }
 
    const size_t target = m_TimeAccountingState.beginBatch();
    Scheduler::instance().drainDeferredTimeAccounting();
    Scheduler::instance().sampleLoadAverage();
    drainDeferredThreadReaps();
    drainAffinityRequests();
    // Sampling can be preempted while owning its global mutex. Keep this
    // worker accounted until every operation in the batch has retired.
    m_TimeAccountingState.finishBatch(target);
 
    // Give ordinary peers a scheduling turn before draining another batch.
    Scheduler::instance().yield();
  }
 
  return 0;
}
 
void PerProcessorScheduler::publishDeferredThreadReap(Thread* thread) {
  const size_t admission = (m_DeferredThreadReapPublicationState += 1);
  if (admission & DeferredReapPublicationClosed) {
    m_DeferredThreadReapPublicationState -= 1;
    FATAL_NOLOCK("Deferred Thread reap publication reached a stopped worker.");
  }
 
  DeferredThreadReapNode& node = thread->m_DeferredReapNode;
  if (node.thread != thread) {
    FATAL_NOLOCK("Deferred Thread reap node has invalid ownership.");
  }
 
  // Make the worker wake edge visible before the node is consumable. A
  // transient pop simply leaves the nonzero count visible for its next turn.
  m_nDeferredThreadReaps += 1;
  m_DeferredThreadReaps.push(node);
  ringIrqWorkDoorbell(m_TimeAccountingWorkerWake);
  m_DeferredThreadReapPublicationState -= 1;
}
 
bool PerProcessorScheduler::drainDeferredThreadReaps() {
  using PopResult =
      IntrusiveMpscQueue<DeferredThreadReapNode, &DeferredThreadReapNode::next>::PopResult;
 
  while (true) {
    DeferredThreadReapNode* node = nullptr;
    const PopResult result = m_DeferredThreadReaps.pop(node);
    if (result == PopResult::Empty) {
      return true;
    }
    if (result == PopResult::Transient) {
      return false;
    }
    if (!node || !node->thread) {
      FATAL("Deferred Thread reap queue returned an invalid target.");
    }
 
    Thread* thread = node->thread;
    Process* process = thread->getParent();
    if (Processor::executionContext() != ExecutionContext::WaitableThread ||
        !Processor::getInterrupts()) {
      FATAL("Deferred Thread reap worker is not at an IRQ-enabled WaitableThread boundary.");
    }
 
    delete thread;
    process->m_DeferredThreadReaps.leave();
    m_nDeferredThreadReaps -= 1;
#if HOSTED && PEDIGREE_HOSTED_SMOKE_TESTS
    m_nDeferredThreadReapCompletions += 1;
#endif
  }
}
 
struct newThreadData {
  Thread* pThread;
  Thread::ThreadStartFunc pStartFunction;
  void* pParam;
  bool bUsermode;
  void* pStack;
  SyscallState state;
  bool useSyscallState;
};
 
void PerProcessorScheduler::startNewThreadWorker(Process* pParent) {
  m_NewThreadDataLock.acquire();
  if (m_NewThreadWorker || m_NewThreadData.count() || m_DelayedNewThreadData.count()) {
    m_NewThreadDataLock.release();
    FATAL("Per-processor thread add worker started with live state.");
  }
  m_StopNewThreadWorker = false;
  m_NewThreadAdmissionOpen = true;
  m_NewThreadDataLock.release();
 
  Thread* pAddThread =
      new Thread(pParent, processorAddThread, reinterpret_cast<void*>(this), 0, false, true);
  pAddThread->setName("PerProcessorScheduler thread add worker");
  m_NewThreadWorker.adopt(pAddThread);
}
 
void PerProcessorScheduler::stopNewThreadWorker() {
  m_NewThreadDataLock.acquire();
  m_NewThreadAdmissionOpen = false;
  m_StopNewThreadWorker = true;
  while (m_DelayedNewThreadData.count()) {
    m_NewThreadData.pushBack(m_DelayedNewThreadData.popFront());
  }
  const bool pending = m_NewThreadData.count();
  m_NewThreadDataLock.release();
 
  if (!m_NewThreadWorker) {
    if (pending) {
      FATAL("Per-processor thread add queue has no worker.");
    }
    return;
  }
 
  m_NewThreadDataCondition.broadcast();
  m_NewThreadWorker.join();
 
  m_NewThreadDataLock.acquire();
  const bool drained = !m_NewThreadData.count() && !m_DelayedNewThreadData.count();
  m_NewThreadDataLock.release();
  if (!drained) {
    FATAL("Per-processor thread add worker stopped before draining.");
  }
}
 
int PerProcessorScheduler::processorAddThread(void* instance) {
  PerProcessorScheduler* pInstance = reinterpret_cast<PerProcessorScheduler*>(instance);
  while (true) {
    pInstance->m_NewThreadDataLock.acquire();
    while (!pInstance->m_NewThreadData.count()) {
      if (pInstance->m_StopNewThreadWorker) {
        pInstance->m_NewThreadDataLock.release();
        return 0;
      }
 
      pInstance->m_NewThreadDataCondition.waitForCompletion(pInstance->m_NewThreadDataLock);
    }
 
    void* p = pInstance->m_NewThreadData.popFront();
 
    newThreadData* pData = reinterpret_cast<newThreadData*>(p);
 
    if (pInstance != &Processor::information().getScheduler()) {
      pInstance->m_NewThreadDataLock.release();
      FATAL("instance " << instance << " does not match current scheduler in processorAddThread!");
    }
 
    Thread* pThread = pData->pThread;
    pThread->m_Lock.acquire();
    const bool retireBeforeStart = pThread->getUnwindState() == Thread::TerminateThread;
    if (pThread->m_Status == Thread::Created && pThread->m_bStartRequested && !retireBeforeStart) {
      pThread->m_bStartRequested = false;
      pThread->m_Status = Thread::Ready;
    }
 
    const bool runnable =
        pThread->m_Status == Thread::Running || pThread->m_Status == Thread::Ready;
    if (retireBeforeStart) {
      pThread->m_Lock.release();
      pInstance->m_NewThreadDataLock.release();
      // This thread has never owned a running stack. The add worker owns
      // the last queued reference and can complete its off-stack exit.
      delete pData;
      pThread->shutdown();
      pThread->m_Lock.acquire();
      EMIT_IF(TRACK_LOCKS) {
        g_LocksCommand.lockReleased(&pThread->m_Lock);
      }
      deleteThread(pThread);
      continue;
    }
 
    if (!runnable) {
      if (pThread->m_Status != Thread::Created) {
        pThread->m_Lock.release();
        pInstance->m_NewThreadDataLock.release();
        FATAL(
            "Per-processor add worker cannot park an already "
            "scheduled thread.");
      }
 
      // Keep the queue and thread state serialized until the parked record
      // is visible, so start or termination cannot miss its publication.
      const bool stopping = pInstance->m_StopNewThreadWorker;
      if (!stopping) {
        pInstance->m_DelayedNewThreadData.pushBack(p);
      }
      pThread->m_Lock.release();
      pInstance->m_NewThreadDataLock.release();
      if (!stopping) {
        continue;
      }
 
      pThread->setUnwindState(Thread::TerminateThread);
      delete pData;
      pThread->shutdown();
      pThread->m_Lock.acquire();
      EMIT_IF(TRACK_LOCKS) {
        g_LocksCommand.lockReleased(&pThread->m_Lock);
      }
      deleteThread(pThread);
      continue;
    }
 
    pInstance->m_NewThreadDataLock.release();
    pThread->setCpuId(Processor::id());
    if (pData->useSyscallState) {
      pInstance->addThread(pThread, pData->state);
    } else {
      pInstance->addThread(pThread, pData->pStartFunction, pData->pParam, pData->bUsermode,
                           pData->pStack);
    }
    delete pData;
  }
}
 
void PerProcessorScheduler::initialise(Thread* pThread) {
  m_LogicalCpu = Processor::index();
  m_PhysicalCpu = Processor::id();
  // Bootstrap identity accessors still return zero here. The discovered
  // topology already distinguishes its logical slot from the firmware ID.
  for (size_t cpu = 0; cpu < Processor::getCount(); ++cpu) {
    ProcessorInformation* information = Processor::informationAt(cpu);
    if (information && &information->getScheduler() == this) {
      m_LogicalCpu = cpu;
      m_PhysicalCpu = information->processorId();
      break;
    }
  }
  m_pSchedulingAlgorithm = new RoundRobin();
 
  if (!pThread->m_Placement.migratable) {
    pThread->m_Placement.allowed = CpuAffinityMask();
    pThread->m_Placement.allowed.set(m_LogicalCpu);
  }
  pThread->m_HasSchedulerContext = true;
  pThread->setStatus(Thread::Running);
  pThread->setCpuId(m_PhysicalCpu);
  Processor::information().setCurrentThread(pThread);
  pThread->recordTime(CpuTimeMode::Kernel);
 
  m_pSchedulingAlgorithm->addThread(pThread);
  Processor::information().setKernelStack(reinterpret_cast<uintptr_t>(pThread->getKernelStack()));
  Processor::setTlsBase(pThread->getTlsBase());
 
  startNewThreadWorker(pThread->getParent());
  startTimeAccountingWorker(pThread->getParent());
 
  // Do not publish timer-driven exit producers until their allocation-free
  // retirement consumer is live and accepting work.
  SchedulerTimer* pTimer = Machine::instance().getSchedulerTimer();
  if (!pTimer) {
    panic("No scheduler timer present.");
  }
  if (!pTimer->registerHandler(this)) {
    FATAL("Per-processor scheduler timer handler is already owned.");
  }
  m_NominalQuantumNs = pTimer->nominalQuantumNs();
  m_OneShotTimer = pTimer->supportsOneShot();
  if (m_OneShotTimer) {
    if (m_LogicalCpu == 0) {
      m_NextLoadSampleDeadline = Time::getTicksFast() + LoadAverage::PeriodNanoseconds;
    }
    updateOneShotTimer();
  }
}
 
void PerProcessorScheduler::programOneShotTimer() {
  if (!m_OneShotTimer) {
    return;
  }
  uint64_t deadline = m_QuantumDeadline;
  if (m_NextLoadSampleDeadline && (!deadline || m_NextLoadSampleDeadline < deadline)) {
    deadline = m_NextLoadSampleDeadline;
  }
  const uint64_t clockDeadline = m_ClockDeadline.value();
  if (clockDeadline && (!deadline || clockDeadline < deadline)) {
    deadline = clockDeadline;
  }
  SchedulerTimer* timer = Machine::instance().getSchedulerTimer();
  if (deadline) {
    if (!timer->armDeadline(deadline)) {
      FATAL_NOLOCK("Failed to arm the local scheduler deadline.");
    }
  } else {
    timer->disarm();
  }
}
 
void PerProcessorScheduler::setClockDeadline(uint64_t deadline) {
  if (!m_OneShotTimer) {
    return;
  }
  uint64_t previous = m_ClockDeadline.value();
  while (!m_ClockDeadline.compareAndSwap(previous, deadline)) {
    previous = m_ClockDeadline.value();
  }
  if ((!previous && !deadline) || (previous && (!deadline || deadline >= previous))) {
    // An already programmed earlier interrupt will re-evaluate the deadline.
    return;
  }
  if (this == &Processor::information().getScheduler()) {
    const bool interrupts = Processor::getInterrupts();
    Processor::setInterrupts(false);
    programOneShotTimer();
    Processor::setInterrupts(interrupts);
    return;
  }
#if X86_COMMON && MULTIPROCESSOR
  ProcessorInformation* information = Processor::informationAt(m_LogicalCpu);
  assert(information);
  if (!Pc::instance().getLocalApic().interProcessorInterrupt(
          information->localApicId(), IPI_RESCHEDULE_VECTOR, LocalApic::deliveryModeFixed, true,
          false)) {
    FATAL_NOLOCK("Remote clock deadline rearm failed.");
  }
#endif
}
 
void PerProcessorScheduler::updateOneShotTimer() {
  if (!m_OneShotTimer) {
    return;
  }
  const bool interrupts = Processor::getInterrupts();
  Processor::setInterrupts(false);
  Thread* current = Processor::information().getCurrentThread();
  if (current && current != m_pIdleThread && m_pSchedulingAlgorithm->hasReady()) {
    const uint64_t now = Time::getTicksFast();
    m_QuantumDeadline =
        now > ~uint64_t(0) - m_NominalQuantumNs ? ~uint64_t(0) : now + m_NominalQuantumNs;
  } else {
    m_QuantumDeadline = 0;
  }
  programOneShotTimer();
  Processor::setInterrupts(interrupts);
}
 
void PerProcessorScheduler::armLocalQuantumIfNeeded() {
  if (!m_OneShotTimer || m_QuantumDeadline || this != &Processor::information().getScheduler()) {
    return;
  }
  const bool interrupts = Processor::getInterrupts();
  Processor::setInterrupts(false);
  Thread* current = Processor::information().getCurrentThread();
  if (current && current != m_pIdleThread && !m_QuantumDeadline) {
    const uint64_t now = Time::getTicksFast();
    m_QuantumDeadline =
        now > ~uint64_t(0) - m_NominalQuantumNs ? ~uint64_t(0) : now + m_NominalQuantumNs;
    programOneShotTimer();
  }
  Processor::setInterrupts(interrupts);
}
 
void PerProcessorScheduler::schedule(Thread::Status nextStatus, bool dispatchEvents) {
  if (!Processor::guardDeviceHardIrqOperation(DeviceHardIrqOperation::Schedule)) {
    return;
  }
 
  bool bWasInterrupts = Processor::getInterrupts();
  Processor::setInterrupts(false);
 
  // The explicit affinity gate may have moved after its caller selected a
  // scheduler. Resolve the receiver only once IRQs exclude another handoff.
  Processor::information().getScheduler().scheduleWithInterruptState(nextStatus, dispatchEvents,
                                                                     bWasInterrupts);
}
 
void PerProcessorScheduler::scheduleWithInterruptState(Thread::Status nextStatus,
                                                       bool dispatchEvents, bool bWasInterrupts) {
#if PEDIGREE_HOSTED_FUNCTION_PROFILE
  hostedFunctionProfileInvalidate(HostedProfileInvalidation::Schedule);
#endif
  assert(!Processor::getInterrupts());
  ActivityDiagnostics::recordScheduleCall();
  Metrics::increment(Metrics::Counter::Schedule);
 
  Thread* pCurrentThread = Processor::information().getCurrentThread();
  if (!pCurrentThread) {
    FATAL("Missing a current thread in PerProcessorScheduler::schedule!");
  }
 
  bool canServiceWorkerWakeups = bWasInterrupts;
#if HOSTED
  canServiceWorkerWakeups &= !pCurrentThread->getHostedSignalDepth();
#endif
  if (canServiceWorkerWakeups && m_IrqWorkDoorbell.compareAndSwap(1, 0)) {
    serviceWorkerWakeups();
  }
 
  // Grab the current thread's lock.
  pCurrentThread->getLock().acquire();
 
  bool dispatchEvent = false;
  if (nextStatus == Thread::Sleeping) {
    if (!pCurrentThread->hasActiveWaitUnlocked()) {
      if (!pCurrentThread->consumeTerminalWaitCancelledBeforeBlockUnlocked()) {
        FATAL("Scheduler refused a sleep without an active WaitQueue.");
      }
 
      // A terminal cancellation can unlink an already-published waiter
      // before this thread has committed its Sleeping transition. The
      // one-shot handoff above is the only valid unlinked sleep.
      pCurrentThread->getLock().release();
      Processor::setInterrupts(bWasInterrupts);
      return;
    }
 
    // The wait record is published before blockCurrent(). A wake in that
    // window changes it away from Waiting, so it is impossible to commit a
    // stale Sleeping transition.
    dispatchEvent = pCurrentThread->hasDeliverableEventsUnlocked();
    if (dispatchEvent) {
      PerProcessorScheduler* readyScheduler = nullptr;
      pCurrentThread->interruptWaitUnlocked(WaitQueue::WakeReason::Event, readyScheduler);
    }
    if (!pCurrentThread->activeWaitPendingUnlocked() || dispatchEvent) {
      pCurrentThread->getLock().release();
      Processor::setInterrupts(bWasInterrupts);
      return;
    }
  }
 
  // Now attempt to get another thread to run.
  // This will also get the lock for the returned thread.
  Thread* pNextThread = selectNext(pCurrentThread);
  if (pNextThread == 0) {
    ActivityDiagnostics::recordSchedulerIdleFallback(
        pCurrentThread->m_Status == Thread::Ready,
        __atomic_load_n(&pCurrentThread->m_ReadyPublicationPending, __ATOMIC_ACQUIRE));
    // A tick or yield does not make a runnable thread idle. Workers which have
    // no work park themselves on their WaitQueue before reaching this path.
    if (nextStatus == Thread::Ready && pCurrentThread != m_pIdleThread &&
        !pCurrentThread->m_ReadyPublicationPending) {
      pNextThread = pCurrentThread;
    } else if (m_pIdleThread == 0) {
      // The scheduler is still bootstrapping, so spinning is the only
      // available fallback.
      pCurrentThread->getLock().release();
      Processor::setInterrupts(bWasInterrupts);
      return;
    } else {
      // Another CPU may publish ready work after selectNext releases the
      // queue lock. It remains queued for the next scheduling interrupt.
      pNextThread = m_pIdleThread;
      if (pNextThread != pCurrentThread)
        pNextThread->getLock().acquire();
    }
  }
 
  // Saving and restoring the same hosted context does not yield and can
  // strand the add-thread worker, so return directly when current stays on CPU.
  if (pNextThread == pCurrentThread) {
    ActivityDiagnostics::recordSameThreadSelection();
    Metrics::increment(Metrics::Counter::SameThread);
    updateOneShotTimer();
    const bool waitOwnsEventDispatch = pCurrentThread->hasActiveWaitUnlocked();
    pCurrentThread->getLock().release();
    Processor::setInterrupts(bWasInterrupts);
    if (dispatchEvents && !waitOwnsEventDispatch) {
      Processor::information().getScheduler().checkEventState(0);
    }
    return;
  }
 
  EMIT_IF(VerboseScheduler) {
    NOTICE_NOLOCK("schedule: " << pCurrentThread << " -> " << pNextThread << " -- "
                               << pCurrentThread->getName() << " -> " << pNextThread->getName());
  }
 
  // Now neither thread can be moved, we're safe to switch.
  ActivityDiagnostics::recordContextSwitch();
  Metrics::increment(Metrics::Counter::ContextSwitch);
  if (pNextThread == m_pIdleThread) {
    ActivityDiagnostics::recordIdleSelection();
    Metrics::increment(Metrics::Counter::IdleSelection);
  }
  if (pCurrentThread != m_pIdleThread)
    pCurrentThread->setStatusUnlocked(nextStatus);
  pNextThread->setStatusUnlocked(Thread::Running);
  Processor::information().setCurrentThread(pNextThread);
  updateOneShotTimer();
 
  // Load the new kernel stack into the TSS, and the new TLS base and switch
  // address spaces
  Processor::information().setKernelStack(
      reinterpret_cast<uintptr_t>(pNextThread->getKernelStack()));
  Processor::switchAddressSpace(*pNextThread->getParent()->getAddressSpace());
  Processor::setTlsBase(pNextThread->getTlsBase());
 
  // Update times.
  pCurrentThread->trackTime(CpuTimeMode::Kernel);
  pNextThread->recordTime(CpuTimeMode::Kernel);
 
  pNextThread->getLock().release();
 
  // The real switch releases the old current thread's lock after changing
  // stacks, so retire that deferred release from the lock checker now.
  EMIT_IF(TRACK_LOCKS) {
    g_LocksCommand.lockReleased(&pCurrentThread->getLock());
  }
 
  EMIT_IF(TRACK_LOCKS) {
    if (!g_LocksCommand.checkSchedule()) {
      FATAL("Lock checker disallowed this reschedule.");
    }
  }
 
  EMIT_IF(SYSTEM_REQUIRES_ATOMIC_CONTEXT_SWITCH) {
    Processor::switchState(bWasInterrupts, pCurrentThread->state(), pNextThread->state(),
                           pCurrentThread->getLock().deferredReleaseWord());
    const bool waitOwnsEventDispatch = pCurrentThread->hasActiveWaitUnlocked();
#if HOSTED && PEDIGREE_HOSTED_SMOKE_TESTS
    Processor::notifyHostedContextSwitchStage(
        ProcessorBase::HostedContextSwitchStage::SchedulerBookkeepingComplete);
    Processor::notifyHostedContextSwitchStage(
        ProcessorBase::HostedContextSwitchStage::SchedulerRestoringInterrupts);
#endif
    Processor::setInterrupts(bWasInterrupts);
    if (dispatchEvents && !waitOwnsEventDispatch) {
      Processor::information().getScheduler().checkEventState(0);
    }
  }
  else {
    // NOTICE_NOLOCK("calling saveState [schedule]");
    if (Processor::saveState(pCurrentThread->state())) {
      // Just context-restored, return.
 
      // A resumed WaitQueue must retire its outer wait record before an
      // event handler can enter another blocking operation. WaitQueue
      // performs the event check immediately after that retirement.
      const bool waitOwnsEventDispatch = pCurrentThread->hasActiveWaitUnlocked();
 
      // Return to previous interrupt state.
      Processor::setInterrupts(bWasInterrupts);
      if (dispatchEvents && !waitOwnsEventDispatch) {
        // We don't have a user-mode stack available here, so pass zero
        // and don't execute user-mode event handlers.
        Processor::information().getScheduler().checkEventState(0);
      }
 
      return;
    }
 
    // Restore context, releasing the old thread's lock when we've switched
    // stacks.
    Processor::restoreState(pNextThread->state(), pCurrentThread->getLock().deferredReleaseWord());
    // Not reached.
  }
}
 
void PerProcessorScheduler::checkEventState(uintptr_t userStack) {
  checkEventState(userStack, Thread::EventSelection::WithoutExactUserReturn);
}
 
void PerProcessorScheduler::checkEventState(uintptr_t userStack, Thread::EventSelection selection) {
  checkEventState(userStack, selection, nullptr, nullptr);
}
 
void PerProcessorScheduler::checkEventState(uintptr_t userStack, Thread::EventSelection selection,
                                            InterruptState* interruptState,
                                            SyscallState* syscallState) {
  bool bWasInterrupts = Processor::getInterrupts();
  Processor::setInterrupts(false);
 
  size_t pageSz = PhysicalMemoryManager::getPageSize();
 
  Thread* pThread = Processor::information().getCurrentThread();
  if (!pThread) {
    Processor::setInterrupts(bWasInterrupts);
    return;
  }
 
  if (pThread->getScheduler() != this) {
    // Wrong scheduler - don't try to run an event for this thread.
    Processor::setInterrupts(bWasInterrupts);
    return;
  }
 
  if (pThread->eventsDeferred()) {
    // Cannot check for any events - we aren't allowed to handle them.
    Processor::setInterrupts(bWasInterrupts);
    return;
  }
 
  if (selection == Thread::EventSelection::WithoutExactUserReturn) {
    pThread->markDeferredUserReturnSignalInterruption();
  }
 
  Event::Delivery eventDelivery = pThread->getNextEvent(selection);
  Event* pEvent = eventDelivery.get();
  if (!eventDelivery) {
    Processor::setInterrupts(bWasInterrupts);
    return;
  }
 
  UserReturnFrame* frame = pThread->currentUserReturnFrame();
  Subsystem* subsystem = pThread->getParent() ? pThread->getParent()->getSubsystem() : nullptr;
  if (frame && subsystem && pEvent->isSignalEvent() && pEvent->getNumber() != 9) {
    if (!bWasInterrupts)
      FATAL_NOLOCK("User-return event interception requires IRQ-enabled entry");
    Subsystem::UserReturnEventResult intercepted;
    {
      // A tracing stop may be killed without returning through this stack.
      Thread::StackDiscardScope deliveryDiscard(
          [](void* value) { static_cast<Event::Delivery*>(value)->reset(); }, &eventDelivery);
      Processor::setInterrupts(true);
      intercepted = subsystem->userReturnEvent(*pThread, *pEvent, *frame);
      Processor::setInterrupts(false);
    }
    if (intercepted != Subsystem::UserReturnEventResult::Deliver) {
      frame->m_Terminal = intercepted == Subsystem::UserReturnEventResult::Terminal;
      eventDelivery.reset();
      Processor::setInterrupts(bWasInterrupts);
      return;
    }
  }
 
  if (pEvent->requiresExactUserReturnState()) {
    Event::UserReturnDelivery result = Event::UserReturnDelivery::NotApplicable;
    if (interruptState || syscallState) {
      if (!bWasInterrupts) {
        FATAL_NOLOCK("Exact user-return event delivery requires an IRQ-enabled thread boundary.");
      }
 
      // Signal-frame construction uses the guarded user-copy path, which may
      // wait for a VM operation already in flight. The Delivery lease pins the
      // event while the raw return frame stays on this thread's kernel stack.
      Processor::setInterrupts(true);
      if (interruptState) {
        result = pEvent->deliverAtUserReturn(*interruptState);
      } else {
        result = pEvent->deliverAtUserReturn(*syscallState);
      }
      Processor::setInterrupts(false);
    }
 
    if (result == Event::UserReturnDelivery::NotApplicable) {
      // Keep the delivery pending until an architecture return tail supplies
      // the exact user register image it needs.
      pThread->sendEvent(pEvent);
    }
    eventDelivery.reset();
    Processor::setInterrupts(bWasInterrupts);
    return;
  }
 
  uintptr_t handlerAddress = pEvent->getHandlerAddress();
 
  VirtualAddressSpace& va = Processor::information().getVirtualAddressSpace();
  physical_uintptr_t page = 0;
  size_t flags = 0;
  bool mappingAvailable = true;
#if HOSTED
  if (pEvent->getHandlerPrivilege() == Event::HandlerPrivilege::Kernel) {
    flags = VirtualAddressSpace::KernelMode;
  } else {
    mappingAvailable = va.isMapped(reinterpret_cast<void*>(handlerAddress));
    if (mappingAvailable) {
      va.getMapping(reinterpret_cast<void*>(handlerAddress), page, flags);
    }
  }
#else
  mappingAvailable = va.isMapped(reinterpret_cast<void*>(handlerAddress));
  if (mappingAvailable) {
    va.getMapping(reinterpret_cast<void*>(handlerAddress), page, flags);
  }
#endif
 
  const bool userHandler = pEvent->getHandlerPrivilege() == Event::HandlerPrivilege::User;
  const bool userAddress =
      !userHandler ||
      (handlerAddress >= va.getUserStart() && handlerAddress < va.getKernelStart() &&
       va.isAddressValid(reinterpret_cast<void*>(handlerAddress)));
  if (!mappingAvailable || !userAddress || !pEvent->isValidHandlerMapping(flags)) {
    ERROR_NOLOCK("checkEventState: Handler address "
                 << Hex << handlerAddress << " does not match its declared "
                 << (userHandler ? "user" : "kernel") << " privilege.");
    Processor::setInterrupts(bWasInterrupts);
    return;
  }
 
  bool alternateStackCandidate = false;
  uintptr_t alternateStackTop = 0;
  if (userHandler && pEvent->prefersAlternateUserStack()) {
    Thread::AlternateSignalStack& alternate = pThread->m_AlternateSignalStack;
    if (alternate.enabled && !alternate.inUse && alternate.base &&
        alternate.size <= (~static_cast<uintptr_t>(0) - alternate.base)) {
      alternateStackTop = (alternate.base + alternate.size) & ~static_cast<uintptr_t>(0xF);
      alternateStackCandidate = true;
    }
  }
 
  SchedulerState* oldState = pThread->pushState();
  if (!oldState) {
    // Keep the event pending until an outer handler unwinds and makes a
    // state slot available.
    pThread->sendEvent(pEvent);
    Processor::setInterrupts(bWasInterrupts);
    return;
  }
 
  if (userHandler) {
    if (alternateStackCandidate) {
      userStack = alternateStackTop;
    }
 
    bool usableUserStack = false;
    if (userStack >= pageSz) {
      const uintptr_t stackPage = userStack - pageSz;
      if (stackPage >= va.getUserStart() && stackPage < va.getKernelStart() &&
          va.isAddressValid(reinterpret_cast<void*>(stackPage)) &&
          va.isMapped(reinterpret_cast<void*>(stackPage))) {
        va.getMapping(reinterpret_cast<void*>(stackPage), page, flags);
        usableUserStack = !(flags & VirtualAddressSpace::KernelMode);
      }
    }
 
    if (usableUserStack && alternateStackCandidate) {
      pThread->m_AlternateSignalStack.inUse = true;
      pThread->m_StateLevels[pThread->m_nStateLevel].m_bOwnsAlternateSignalStack = true;
    }
 
    if (!usableUserStack) {
      VirtualAddressSpace::Stack* stateStack = pThread->getStateUserStack();
      if (!stateStack || !va.isMapped(adjust_pointer(stateStack->getTop(), -pageSz))) {
        stateStack = va.allocateStack();
        if (!stateStack)
          panic("checkEventState: no user fallback stack");
        pThread->setStateUserStack(stateStack);
      }
 
      userStack = reinterpret_cast<uintptr_t>(stateStack->getTop());
    }
  }
 
  // The address of the serialize buffer is determined by the thread ID and
  // the nesting level.
  uintptr_t addr = Event::getHandlerBuffer() +
                   (pThread->getId() * MAX_NESTED_EVENTS + (pThread->getStateLevel() - 1)) *
                       Event::getHandlerBufferSize();
 
  // Ensure the complete serialized-event span is mapped.
  for (size_t offset = 0; offset < Event::getHandlerBufferSize(); offset += pageSz) {
    void* eventPage = reinterpret_cast<void*>(addr + offset);
    if (!va.isMapped(eventPage)) {
      physical_uintptr_t p = PhysicalMemoryManager::instance().allocatePage();
      if (!p) {
        panic("checkEventState: Out of memory!");
      }
      va.map(p, eventPage, VirtualAddressSpace::Write | VirtualAddressSpace::RuntimeMapping);
    }
  }
 
  const bool deletableEvent = pEvent->isDeletable();
  if (!deletableEvent && !userHandler) {
    eventDelivery.beginDispatch();
  }
  pEvent->serialize(reinterpret_cast<uint8_t*>(addr));
 
  // Fire-and-forget events are fully represented by their serialized data.
  // Stable kernel events retain the lease through their callback because
  // that callback may intentionally refer back to the original object.
  if (deletableEvent) {
    eventDelivery.reset();
  }
 
  EMIT_IF(!SYSTEM_REQUIRES_ATOMIC_CONTEXT_SWITCH) {
    if (Processor::saveState(*oldState)) {
      // Just context-restored.
      Processor::setInterrupts(bWasInterrupts);
      return;
    }
  }
 
  if (!userHandler) {
    // Setup must be atomic, but the callback is ordinary thread work. In
    // particular, a return-to-user interrupt tail enters here only after
    // raw handler/accounting scopes have unwound and with IRQs enabled.
    Processor::setInterrupts(bWasInterrupts);
#if HOSTED
    // Hosted state levels use distinct signal/scheduler stacks. Run the
    // handler on the selected level so an interrupt cannot save a
    // level-one frame that is physically still on level zero.
    callOnStack(reinterpret_cast<uintptr_t>(pThread->getKernelStack()), handlerAddress, addr);
#else
    void (*fn)(size_t) = reinterpret_cast<void (*)(size_t)>(handlerAddress);
    fn(addr);
#endif
 
    Processor::setInterrupts(false);
    eventDelivery.reset();
    pThread->popState(!HOSTED);
    Processor::setInterrupts(bWasInterrupts);
    return;
  } else if (userStack != 0) {
    // User delivery consumes only the serialized representation.
    eventDelivery.reset();
    pThread->transitionTime(CpuTimeMode::Kernel, CpuTimeMode::User);
    EMIT_IF(SYSTEM_REQUIRES_ATOMIC_CONTEXT_SWITCH) {
      Processor::saveAndJumpUser(bWasInterrupts, *oldState, 0, Event::getTrampoline(), userStack,
                                 handlerAddress, addr);
      Processor::setInterrupts(bWasInterrupts);
    }
    else {
      Processor::jumpUser(0, Event::getTrampoline(), userStack, handlerAddress, addr);
      // Not reached.
    }
  }
}
 
void PerProcessorScheduler::eventHandlerReturned() {
  Processor::setInterrupts(false);
 
  Thread* pThread = Processor::information().getCurrentThread();
  pThread->abandonCurrentState(false);
 
  Processor::restoreState(pThread->state());
  // Not reached.
}
 
void PerProcessorScheduler::addThread(Thread* pThread, Thread::ThreadStartFunc pStartFunction,
                                      void* pParam, bool bUsermode, void* pStack) {
  // Handle wrong CPU, and handle thread not yet ready to schedule.
  if (this != &Processor::information().getScheduler() || pThread->getStatus() == Thread::Created) {
    newThreadData* pData = new newThreadData;
    pData->pThread = pThread;
    pData->pStartFunction = pStartFunction;
    pData->pParam = pParam;
    pData->bUsermode = bUsermode;
    pData->pStack = pStack;
    pData->useSyscallState = false;
 
    pThread->m_Lock.release();
 
    m_NewThreadDataLock.acquire();
    if (!m_NewThreadAdmissionOpen) {
      m_NewThreadDataLock.release();
      delete pData;
      panic("Thread admitted after its per-processor worker stopped.");
    }
    m_NewThreadData.pushBack(pData);
    m_NewThreadDataLock.release();
 
    m_NewThreadDataCondition.signal();
 
    return;
  }
 
  pThread->setCpuId(Processor::id());
  pThread->setScheduler(this);
 
  bool bWasInterrupts = Processor::getInterrupts();
  Processor::setInterrupts(false);
 
  // We assume here that pThread's lock is already taken.
 
  Thread* pCurrentThread = Processor::information().getCurrentThread();
 
  // Grab the current thread's lock.
  pCurrentThread->getLock().acquire();
 
  m_pSchedulingAlgorithm->addThread(pThread);
 
  assert(pThread->m_Placement.allowed.contains(m_LogicalCpu));
  pThread->m_HasSchedulerContext = true;
  // Now neither thread can be moved, we're safe to switch.
  Metrics::increment(Metrics::Counter::ContextSwitch);
  if (pThread == m_pIdleThread) {
    Metrics::increment(Metrics::Counter::IdleSelection);
  }
  if (pCurrentThread != m_pIdleThread) {
    pCurrentThread->setStatusUnlocked(Thread::Ready);
  }
  pThread->setStatusUnlocked(Thread::Running);
  Processor::information().setCurrentThread(pThread);
  updateOneShotTimer();
  void* kernelStack = pThread->getKernelStack();
  Processor::information().setKernelStack(reinterpret_cast<uintptr_t>(kernelStack));
  Processor::switchAddressSpace(*pThread->getParent()->getAddressSpace());
  Processor::setTlsBase(pThread->getTlsBase());
 
  // This thread is safe from being moved as its status is now "running".
  // It is worth noting that we can't just call exit() here, as the lock is
  // not necessarily actually taken.
  if (pThread->getLock().interrupts())
    bWasInterrupts = true;
  bool bWas = pThread->getLock().acquired();
  pThread->getLock().unlockForScheduler();
  EMIT_IF(TRACK_LOCKS) {
    // Satisfy the lock checker; we're releasing these out of order, so make
    // sure the checker sees them unlocked in order.
    g_LocksCommand.lockReleased(&pCurrentThread->getLock());
    if (bWas) {
      // Lock was in fact locked before.
      g_LocksCommand.lockReleased(&pThread->getLock());
    }
    if (!g_LocksCommand.checkSchedule()) {
      FATAL("Lock checker disallowed this reschedule.");
    }
  }
 
  pCurrentThread->trackTime(CpuTimeMode::Kernel);
  pThread->recordTime(bUsermode ? CpuTimeMode::User : CpuTimeMode::Kernel);
 
  EMIT_IF(SYSTEM_REQUIRES_ATOMIC_CONTEXT_SWITCH) {
    if (bUsermode) {
      Processor::saveAndJumpUser(
          bWasInterrupts, pCurrentThread->state(), pCurrentThread->getLock().deferredReleaseWord(),
          reinterpret_cast<uintptr_t>(pStartFunction), reinterpret_cast<uintptr_t>(pStack),
          reinterpret_cast<uintptr_t>(pParam));
    } else {
      Processor::saveAndJumpKernel(
          bWasInterrupts, pCurrentThread->state(), pCurrentThread->getLock().deferredReleaseWord(),
          reinterpret_cast<uintptr_t>(pStartFunction), reinterpret_cast<uintptr_t>(pStack),
          reinterpret_cast<uintptr_t>(pParam));
    }
    Processor::setInterrupts(bWasInterrupts);
  }
  else {
    if (Processor::saveState(pCurrentThread->state())) {
      // Just context-restored.
      if (bWasInterrupts)
        Processor::setInterrupts(true);
      return;
    }
 
    if (bUsermode) {
      Processor::jumpUser(pCurrentThread->getLock().deferredReleaseWord(),
                          reinterpret_cast<uintptr_t>(pStartFunction),
                          reinterpret_cast<uintptr_t>(pStack), reinterpret_cast<uintptr_t>(pParam));
    } else {
      Processor::jumpKernel(pCurrentThread->getLock().deferredReleaseWord(),
                            reinterpret_cast<uintptr_t>(pStartFunction),
                            reinterpret_cast<uintptr_t>(pStack),
                            reinterpret_cast<uintptr_t>(pParam));
    }
  }
}
 
void PerProcessorScheduler::addThread(Thread* pThread, SyscallState& state) {
  // Handle wrong CPU, and handle thread not yet ready to schedule.
  if (this != &Processor::information().getScheduler() || pThread->getStatus() == Thread::Created) {
    newThreadData* pData = new newThreadData;
    pData->pThread = pThread;
    pData->pParam = nullptr;
    pData->useSyscallState = true;
    pData->state = state;
 
    pThread->m_Lock.release();
 
    m_NewThreadDataLock.acquire();
    if (!m_NewThreadAdmissionOpen) {
      m_NewThreadDataLock.release();
      delete pData;
      panic("Thread admitted after its per-processor worker stopped.");
    }
    m_NewThreadData.pushBack(pData);
    m_NewThreadDataLock.release();
 
    m_NewThreadDataCondition.signal();
    return;
  }
 
  pThread->setCpuId(Processor::id());
  pThread->setScheduler(this);
 
  bool bWasInterrupts = Processor::getInterrupts();
  Processor::setInterrupts(false);
 
  // We assume here that pThread's lock is already taken.
 
  Thread* pCurrentThread = Processor::information().getCurrentThread();
 
  // Grab the current thread's lock.
  pCurrentThread->getLock().acquire();
 
  m_pSchedulingAlgorithm->addThread(pThread);
 
  assert(pThread->m_Placement.allowed.contains(m_LogicalCpu));
  pThread->m_HasSchedulerContext = true;
  // Now neither thread can be moved, we're safe to switch.
  Metrics::increment(Metrics::Counter::ContextSwitch);
  if (pThread == m_pIdleThread) {
    Metrics::increment(Metrics::Counter::IdleSelection);
  }
 
  if (pCurrentThread != m_pIdleThread) {
    pCurrentThread->setStatusUnlocked(Thread::Ready);
  }
  pThread->setStatusUnlocked(Thread::Running);
  Processor::information().setCurrentThread(pThread);
  updateOneShotTimer();
  void* kernelStack = pThread->getKernelStack();
  Processor::information().setKernelStack(reinterpret_cast<uintptr_t>(kernelStack));
  Processor::switchAddressSpace(*pThread->getParent()->getAddressSpace());
  Processor::setTlsBase(pThread->getTlsBase());
 
  // This thread is safe from being moved as its status is now "running".
  // It is worth noting that we can't just call exit() here, as the lock is
  // not necessarily actually taken.
  if (pThread->getLock().interrupts())
    bWasInterrupts = true;
  bool bWas = pThread->getLock().acquired();
  pThread->getLock().unlockForScheduler();
  EMIT_IF(TRACK_LOCKS) {
    g_LocksCommand.lockReleased(&pCurrentThread->getLock());
    if (bWas) {
      // We unlocked the lock, so track that unlock.
      g_LocksCommand.lockReleased(&pThread->getLock());
    }
    if (!g_LocksCommand.checkSchedule()) {
      FATAL("Lock checker disallowed this reschedule.");
    }
  }
 
#if X64 && !HOSTED
  // CLONE_SETTLS may have replaced the copied parent's base above.
  state.refreshUserTlsBase();
#endif
 
  // Copy the SyscallState into this thread's kernel stack.
  uintptr_t kStack = reinterpret_cast<uintptr_t>(pThread->getKernelStack());
  kStack -= sizeof(SyscallState);
  MemoryCopy(reinterpret_cast<void*>(kStack), reinterpret_cast<void*>(&state),
             sizeof(SyscallState));
 
  // Grab a reference to the stack in the form of a full SyscallState.
  SyscallState& newState = *reinterpret_cast<SyscallState*>(kStack);
 
  pCurrentThread->trackTime(CpuTimeMode::Kernel);
  // restoreState(SyscallState) returns directly to the forked userspace
  // frame, so establish its user baseline before the no-return handoff.
  pThread->recordTime(CpuTimeMode::User);
 
  EMIT_IF(SYSTEM_REQUIRES_ATOMIC_CONTEXT_SWITCH) {
    NOTICE("restoring (new) syscall state");
    Processor::switchState(bWasInterrupts, pCurrentThread->state(), newState,
                           pCurrentThread->getLock().deferredReleaseWord());
  }
  else {
    if (Processor::saveState(pCurrentThread->state())) {
      // Just context-restored.
      if (bWasInterrupts)
        Processor::setInterrupts(true);
      return;
    }
 
    Processor::restoreState(newState, pCurrentThread->getLock().deferredReleaseWord());
  }
}
 
void PerProcessorScheduler::commitCurrentThreadExit(Spinlock* pLock) {
  Thread* pThread = Processor::information().getCurrentThread();
  if (!pThread) {
    FATAL("Clean thread exit has no current Thread.");
  }
 
  if (__atomic_load_n(&pThread->m_TerminationDeferralDepth, __ATOMIC_ACQUIRE) ||
      __atomic_load_n(&pThread->m_EventDeferralDepth, __ATOMIC_ACQUIRE)) {
    FATAL("Clean thread exit reached a boundary with active deferral scopes.");
  }
 
  for (size_t level = 0; level < MAX_NESTED_EVENTS; ++level) {
    if (__atomic_load_n(&pThread->m_pDeferredScopes[level], __ATOMIC_ACQUIRE)) {
      FATAL("Clean thread exit reached a boundary with armed cleanup records.");
    }
  }
  if (pThread->hasActiveWaitAtAnyLevel()) {
    FATAL("Clean thread exit reached a boundary with an active WaitQueue record.");
  }
 
  const bool transferToIdle = pThread && pThread->m_ExitToIdle;
  pThread->m_ExitToIdle = false;
  finishCurrentThreadExit(pLock, transferToIdle);
}
 
void PerProcessorScheduler::killCurrentThread(Spinlock* pLock) {
  abandonCurrentThreadStack(StackDiscardReason::LegacyAbiCall, pLock);
}
 
void PerProcessorScheduler::abandonCurrentThreadStack(StackDiscardReason reason, Spinlock* pLock) {
  Thread* pThread = Processor::information().getCurrentThread();
  if (!pThread) {
    FATAL("Stack discard has no current Thread.");
  }
 
  g_StackDiscardCount += 1;
  switch (reason) {
    case StackDiscardReason::EmergencyProcessKill:
      g_EmergencyProcessKillDiscardCount += 1;
      break;
    case StackDiscardReason::HostedRegression:
      g_HostedRegressionDiscardCount += 1;
      break;
    case StackDiscardReason::LegacyAbiCall:
      g_LegacyAbiDiscardCount += 1;
      break;
  }
 
  // Cleanup callbacks can destroy the object which owns a published wait
  // queue. Unlink intrusive wait records before invoking those callbacks.
  pThread->unlinkWaitsForStackDiscard();
 
  // No C++ destructors run after this call. Retire registered stack-owned
  // lifetime records while their abandoned storage is still mapped.
  pThread->retireDeferredScopes(true);
 
  const bool transferToIdle = pThread->m_ExitToIdle;
  pThread->m_ExitToIdle = false;
  finishCurrentThreadExit(pLock, transferToIdle);
}
 
size_t PerProcessorScheduler::stackDiscardCount() {
  return g_StackDiscardCount.value();
}
 
size_t PerProcessorScheduler::stackDiscardCount(StackDiscardReason reason) {
  switch (reason) {
    case StackDiscardReason::EmergencyProcessKill:
      return g_EmergencyProcessKillDiscardCount.value();
    case StackDiscardReason::HostedRegression:
      return g_HostedRegressionDiscardCount.value();
    case StackDiscardReason::LegacyAbiCall:
      return g_LegacyAbiDiscardCount.value();
  }
  return 0;
}
 
void PerProcessorScheduler::requestCurrentThreadExitToIdle() {
  Thread* pThread = Processor::information().getCurrentThread();
  if (!pThread || !m_pIdleThread || pThread == m_pIdleThread) {
    panic("Current thread has no distinct idle shutdown owner!");
  }
  pThread->m_ExitToIdle = true;
}
 
void PerProcessorScheduler::finishCurrentThreadExit(Spinlock* pLock, bool transferToIdle) {
  Thread* pThread = Processor::information().getCurrentThread();
 
  // Start shutting down the current thread while we can still schedule it.
  pThread->shutdown();
 
  Processor::setInterrupts(false);
  PerProcessorScheduler& owner = Processor::information().getScheduler();
 
  // Removing the current thread. Grab its lock.
  pThread->getLock().acquire();
 
  // If we're tracking locks, don't pollute the results. Yes, we've kept
  // this lock held, but it no longer matters.
  EMIT_IF(TRACK_LOCKS) {
    g_LocksCommand.lockReleased(&pThread->getLock());
    if (pLock) {
      g_LocksCommand.lockReleased(pLock);
    }
    if (!g_LocksCommand.checkSchedule()) {
      FATAL("Lock checker disallowed this reschedule.");
    }
  }
 
  // Get another thread ready to schedule.
  // This will also get the lock for the returned thread.
  Thread* pNextThread = transferToIdle ? nullptr : owner.selectNext(pThread);
 
  if (transferToIdle && (!owner.m_pIdleThread || owner.m_pIdleThread == pThread)) {
    panic("Current thread has no distinct idle shutdown owner!");
  }
 
  if (pNextThread == 0 && owner.m_pIdleThread == 0) {
    // Nothing to switch to, we're in a VERY bad situation.
    panic("Attempting to kill only thread on this processor!");
  } else if (pNextThread == 0) {
    pNextThread = owner.m_pIdleThread;
    if (pNextThread != pThread)
      pNextThread->getLock().acquire();
  }
 
  Metrics::increment(Metrics::Counter::ContextSwitch);
  if (pNextThread == owner.m_pIdleThread) {
    Metrics::increment(Metrics::Counter::IdleSelection);
  }
  pNextThread->setStatusUnlocked(Thread::Running);
  Processor::information().setCurrentThread(pNextThread);
  owner.updateOneShotTimer();
  void* kernelStack = pNextThread->getKernelStack();
  Processor::information().setKernelStack(reinterpret_cast<uintptr_t>(kernelStack));
  EMIT_IF(!HOSTED) {
    Processor::switchAddressSpace(*pNextThread->getParent()->getAddressSpace());
  }
  Processor::setTlsBase(pNextThread->getTlsBase());
 
  pThread->trackTime(CpuTimeMode::Kernel);
  pNextThread->recordTime(CpuTimeMode::Kernel);
 
  pNextThread->getLock().exit();
 
  // Pass in the lock atom we were given if possible, as the caller wants an
  // atomic release (i.e. once the thread is no longer able to be scheduled).
  deleteThreadThenRestoreState(pThread, pNextThread->state(),
                               pLock ? pLock->deferredReleaseWord() : 0);
}
 
void PerProcessorScheduler::deleteThread(Thread* pThread) {
#if HOSTED
  // A hosted user Thread is still ASan's active fiber until the assembly
  // handoff has moved onto safe_stack. Unmapping its address space earlier
  // makes ASan—and potentially host signal machinery—observe a live stack
  // disappearing underneath the no-return transition.
  Thread* replacement = Processor::information().getCurrentThread();
  if (!replacement || replacement == pThread) {
    FATAL("Hosted Thread deletion has no replacement address space");
  }
  Processor::switchAddressSpace(*replacement->getParent()->getAddressSpace());
#endif
 
  Process* pProcess = pThread->getParent();
  // This runs on a temporary handoff stack before the replacement Thread
  // has retired any WaitQueue record it resumed from. Blocking here would
  // try to enrol that Thread in two queues at once. Close admission now;
  // joins drain in ordinary thread context, while the final external lease
  // completes deferred detached deletion.
  pThread->closeExternalLeaseAdmission();
 
  // This is the scheduler's final access outside the Process-serialised
  // retirement block below. Publish the unlocked atom before m_bReapable so
  // a joiner or detach owner can treat reapable as a true final-use boundary.
  // No Process lock is held here, so waking another same-core thread cannot
  // resume it into a conflicting terminal operation under that lock.
  pThread->getLock().unlockForScheduler();
 
  bool deleteTarget = false;
  bool completesProcessExit = false;
  bool wakeExitOwner = false;
  {
    RecursingLockGuard<Spinlock> processGuard(pProcess->m_Lock);
    deleteTarget = pThread->markReapable();
    completesProcessExit = pProcess->terminatingThreadReapable(pThread, wakeExitOwner);
 
    if (deleteTarget) {
      if (!pProcess->m_DeferredThreadReaps.tryEnter()) {
        FATAL_NOLOCK("Thread retirement raced closed Process admission.");
      }
    }
  }
 
  // Process-exit progress is a predicate update under m_Lock followed by an
  // out-of-lock notification. Keeping the WaitQueue wake outside m_Lock
  // prevents a resumed owner from acquiring the queue under an outer lock.
  if (wakeExitOwner) {
    pProcess->m_TerminationWaiters.wakeAll();
    pProcess->m_ExecWaiters.wakeAll();
  }
 
  if (deleteTarget) {
    PerProcessorScheduler& scheduler = Processor::information().getScheduler();
    if (pThread->getScheduler() && pThread->getScheduler() != &scheduler) {
      FATAL_NOLOCK("Thread retirement reached a non-owning scheduler.");
    }
    scheduler.publishDeferredThreadReap(pThread);
  }
 
  if (!completesProcessExit) {
    return;
  }
 
  // This is the final Process access: publication can wake a reaper that
  // destroys both the Process and its retained, reapable Thread objects.
  pProcess->publishTerminationReapable();
}
 
void PerProcessorScheduler::removeThread(Thread* pThread) {
  m_pSchedulingAlgorithm->removeThread(pThread);
}
 
void PerProcessorScheduler::blockCurrent() {
  schedule(Thread::Sleeping);
}
 
void PerProcessorScheduler::publishReadyFromWait(Thread* pThread) {
  if (!pThread) {
    assert(false);
    return;
  }
  pThread->publishReadyNotification();
}
 
Thread* PerProcessorScheduler::selectNext(Thread* current) {
  // Keep selecting the idle owner until it retires that role itself. A tick
  // can preempt its first resumed turn before it observes the shutdown flag.
  if (__atomic_load_n(&m_IdleWakeRequested, __ATOMIC_ACQUIRE)) {
    Thread* idle = __atomic_load_n(&m_pIdleThread, __ATOMIC_ACQUIRE);
    if (idle) {
      if (idle == current)
        return nullptr;
      idle->m_Lock.acquire();
      return idle;
    }
  }
  while (Thread* candidate = m_pSchedulingAlgorithm->getNext(current)) {
    candidate->m_Lock.acquire();
    if (candidate->getScheduler() == this && candidate->m_Status == Thread::Ready &&
        !candidate->m_ReadyPublicationPending)
      return candidate;
    candidate->m_Lock.release();
  }
  return nullptr;
}
 
void PerProcessorScheduler::timer(uint64_t delta, InterruptState& state) {
  (void)state;
  if constexpr (PEDIGREE_TIME_ACCOUNTING && PEDIGREE_SAMPLED_TIME_ACCOUNTING) {
    Thread* current = Processor::information().getCurrentThread();
    if (current && current != m_pIdleThread && delta) {
      current->accountTimerTick(delta, state.kernelMode());
    }
  }
  ActivityDiagnostics::recordSchedulerTimer();
  Metrics::increment(Metrics::Counter::Timer);
  if (m_OneShotTimer) {
    if (!delta) {
      programOneShotTimer();
      return;
    }
    const uint64_t now = Time::getTicksFast();
    if (m_NextLoadSampleDeadline && now >= m_NextLoadSampleDeadline) {
      m_NextLoadSampleDeadline = now > ~uint64_t(0) - LoadAverage::PeriodNanoseconds
                                     ? ~uint64_t(0)
                                     : now + LoadAverage::PeriodNanoseconds;
      Scheduler::instance().requestLoadAverageSample();
    }
    if (m_QuantumDeadline && now >= m_QuantumDeadline) {
      m_QuantumDeadline = 0;
      m_ReschedulePending = 1;
    }
    const uint64_t clockDeadline = m_ClockDeadline.value();
    if (clockDeadline && now >= clockDeadline && m_ClockDeadline.compareAndSwap(clockDeadline, 0)) {
      Machine::instance().getTimer()->deadlineInterrupt();
    }
    programOneShotTimer();
    return;
  }
  if (delta) {
    Scheduler::instance().requestLoadAverageSample();
  }
  // The raw handler only records the scheduling edge. The architecture IRQ
  // return services it after all hard-handler scopes have unwound.
  if (!delta) {
    m_ReschedulePending = 1;
  } else if (++m_SchedulerTickCounter >= PEDIGREE_SCHEDULER_TICK_DIVISOR) {
    m_SchedulerTickCounter = 0;
    m_ReschedulePending = 1;
  }
}
 
void PerProcessorScheduler::threadStatusChanged(Thread* pThread) {
  bool wakeWorker = false;
  bool queueLocked = false;
  PerProcessorScheduler* readyOwner = nullptr;
  pThread->m_Lock.acquire();
  if (pThread->m_Status == Thread::Created) {
    // Acquiring the queue mutex may schedule its worker on this CPU. Never
    // retain the thread spinlock that the worker needs while waiting for it.
    pThread->m_Lock.release();
    m_NewThreadDataLock.acquire();
    queueLocked = true;
    pThread->m_Lock.acquire();
    if (pThread->m_Status == Thread::Created) {
      for (List<void*>::Iterator it = m_DelayedNewThreadData.begin();
           it != m_DelayedNewThreadData.end();) {
        newThreadData* pData = reinterpret_cast<newThreadData*>(*it);
        if (pData->pThread == pThread) {
          void* p = *it;
          it = m_DelayedNewThreadData.erase(it);
          m_NewThreadData.pushBack(p);
          wakeWorker = true;
        } else {
          ++it;
        }
      }
    }
  }
 
  PerProcessorScheduler* owner = pThread->getScheduler();
  assert(owner);
  owner->m_pSchedulingAlgorithm->threadStatusChanged(pThread);
  if (pThread->m_Status == Thread::Ready && !pThread->m_ReadyPublicationPending) {
    readyOwner = owner;
  }
  pThread->m_Lock.release();
  if (queueLocked) {
    m_NewThreadDataLock.release();
  }
 
  if (wakeWorker) {
    m_NewThreadDataCondition.signal();
  }
  if (readyOwner) {
    readyOwner->prompt(true);
  }
}
 
void PerProcessorScheduler::ringIrqWorkDoorbell() {
  m_IrqWorkDoorbell = 1;
  prompt();
}
 
void PerProcessorScheduler::registerWorkerWake(SchedulerWorkerWake& worker, WaitQueue& waiters) {
  LockGuard<Spinlock> guard(m_IrqWorkLock);
  if (worker.m_pWaiters || worker.m_pNext) {
    FATAL("Scheduler worker wake was registered twice.");
  }
  worker.m_Pending = 0;
  worker.m_pWaiters = &waiters;
  worker.m_pNext = m_pWorkerWakeHead;
  m_pWorkerWakeHead = &worker;
}
 
void PerProcessorScheduler::unregisterWorkerWake(SchedulerWorkerWake& worker) {
  LockGuard<Spinlock> guard(m_IrqWorkLock);
  SchedulerWorkerWake** link = &m_pWorkerWakeHead;
  while (*link && *link != &worker) {
    link = &(*link)->m_pNext;
  }
  if (!*link) {
    FATAL("Scheduler worker wake was not registered.");
  }
  *link = worker.m_pNext;
  worker.m_pNext = nullptr;
  worker.m_pWaiters = nullptr;
  worker.m_Pending = 0;
}
 
void PerProcessorScheduler::ringIrqWorkDoorbell(SchedulerWorkerWake& worker, bool promptOwner) {
  worker.m_Pending = 1;
  if (promptOwner) {
    ringIrqWorkDoorbell();
  } else {
    m_IrqWorkDoorbell = 1;
  }
}
 
void PerProcessorScheduler::serviceWorkerWakeups() {
  LockGuard<Spinlock> guard(m_IrqWorkLock);
  bool retry = false;
  for (SchedulerWorkerWake* worker = m_pWorkerWakeHead; worker; worker = worker->m_pNext) {
    if (!worker->m_Pending.value() || !worker->m_pWaiters) {
      continue;
    }
 
    if (worker->m_pWaiters->wakeOne()) {
      Metrics::increment(Metrics::Counter::WorkerWake);
      worker->m_Pending.compareAndSwap(1, 0);
    } else {
      // A producer may publish between the worker's empty check and its
      // WaitQueue enrollment. Keep the edge armed so the next scheduler
      // boundary retries after the waiter is visible.
      retry = true;
    }
  }
  if (retry) {
    m_IrqWorkDoorbell = 1;
  }
}
 
void PerProcessorScheduler::publishDeferredTimeAccounting() {
  m_TimeAccountingState.publish();
  ringIrqWorkDoorbell(m_TimeAccountingWorkerWake);
}
 
void PerProcessorScheduler::serviceIrqWorkDoorbell() {
  Metrics::increment(Metrics::Counter::RescheduleService);
  if (!m_pSchedulingAlgorithm || !Processor::information().getCurrentThread()) {
    return;
  }
 
  // One bounded claim is enough: a racing ring remains set for the next
  // scheduler tick.
  if (m_IrqWorkDoorbell.compareAndSwap(1, 0)) {
    serviceWorkerWakeups();
    m_ReschedulePending.compareAndSwap(1, 0);
    schedule();
  }
}
 
void PerProcessorScheduler::servicePendingScheduling() {
  Metrics::increment(Metrics::Counter::RescheduleService);
  if (!m_pSchedulingAlgorithm || !Processor::information().getCurrentThread()) {
    return;
  }
 
  m_RemotePromptPending = 0;
  bool pending = m_ReschedulePending.compareAndSwap(1, 0);
  if (m_IrqWorkDoorbell.compareAndSwap(1, 0)) {
    serviceWorkerWakeups();
    pending = true;
    m_ReschedulePending.compareAndSwap(1, 0);
  }
  if (pending) {
    schedule(Thread::Ready, false);
  }
}
 
bool PerProcessorScheduler::serviceProcessStopAtUserReturn(ProcessStopGateMode mode) {
  if (!Processor::getInterrupts() || Processor::inDeviceHardIrq() ||
      Processor::executionContext() != ExecutionContext::WaitableThread) {
    FATAL_NOLOCK(
        "Return-to-user stop work requires an IRQ-enabled thread "
        "boundary.");
  }
 
  Thread* current = Processor::information().getCurrentThread();
  if (current && current->currentTimeAccountingMode() != CpuTimeMode::Kernel) {
    FATAL_NOLOCK("Return-to-user stop work escaped Kernel accounting mode");
  }
  if (!current) {
    return false;
  }
 
  Process* process = current->getParent();
  if (!process) {
    return current->getUnwindState() != Thread::Continue;
  }
 
  while (true) {
    bool dispatchKernelEvent = false;
    Thread::EventSelection selection = Thread::EventSelection::StoppedProcessKernel;
    WaitQueue::WakeReason reason = WaitQueue::WakeReason::Spurious;
    {
      // Signal publication and resume both use this queue, making the
      // predicate check and waiter publication one atomic handshake.
      auto guard = current->m_EventWaiters.acquire();
      {
        LockGuard<Spinlock> threadGuard(current->m_Lock);
        if (current->getUnwindState() != Thread::Continue) {
          return true;
        }
 
        const Process::ProcessState state = process->getState();
        if (state == Process::Active) {
          selection = Thread::EventSelection::KernelDeliverable;
          dispatchKernelEvent = mode == ProcessStopGateMode::DirectUserTransition &&
                                current->hasDeliverableEventsUnlocked(selection);
          if (!dispatchKernelEvent) {
            return false;
          }
        }
        if (state == Process::Terminated || state == Process::Reaped) {
          FATAL_NOLOCK("A live Thread reached userspace after its Process terminated");
        }
 
        // SIGKILL and other kernel events that explicitly permit stopped
        // delivery must run before this thread can wait indefinitely.
        if (state == Process::Suspended) {
          selection = Thread::EventSelection::StoppedProcessKernel;
          dispatchKernelEvent = current->hasDeliverableEventsUnlocked(selection);
        }
      }
 
      if (!dispatchKernelEvent) {
        // A Terminating peer can observe the Process transition just before
        // beginTermination publishes that peer's unwind state. Its terminal
        // wake releases this same wait.
        reason = guard.waitWithoutEventDispatch(
            WaitQueue::Channel(process, static_cast<uintptr_t>(Thread::ProcessWait)),
            Thread::ProcessWait, reinterpret_cast<uintptr_t>(__builtin_return_address(0)));
      }
    }
 
    if (dispatchKernelEvent) {
      // Direct transitions have no reusable user stack. A selection-specific
      // dequeue prevents a racing resume from broadening stopped delivery.
      Processor::information().getScheduler().checkEventState(0, selection);
      continue;
    }
    if (reason == WaitQueue::WakeReason::Terminating ||
        reason == WaitQueue::WakeReason::Unwinding) {
      return true;
    }
  }
}
 
bool PerProcessorScheduler::serviceUserReturnWork(InterruptState& state,
                                                  UserReturnFrame::Origin origin,
                                                  bool diagnosticSample) {
  const uint64_t workStart = diagnosticSample ? ActivityDiagnostics::timestamp() : 0;
  auto finishWork = [diagnosticSample, workStart](bool terminal) {
    if (diagnosticSample) {
      ActivityDiagnostics::recordUserReturnStage(
          ActivityDiagnostics::UserReturnStage::InterruptWork,
          ActivityDiagnostics::timestamp() - workStart);
    }
    return terminal;
  };
  Thread* owner = Processor::information().getCurrentThread();
  if (!owner)
    return finishWork(false);
#if X64 && !HOSTED
  {
    EnsureInterrupts interrupts(false);
    state.setFlags(state.getFlags() | 0x202);
  }
#endif
#if PEDIGREE_FAST_USER_RETURN
  if ((origin == UserReturnFrame::Origin::Syscall ||
       origin == UserReturnFrame::Origin::Interrupt) &&
      owner->canSkipUserReturnWork()) {
    return finishWork(false);
  }
#endif
  UserReturnFrame frame(*owner, state, origin);
  Thread::UserReturnFrameScope frameScope(*owner, frame);
 
  Subsystem* subsystem = owner->getParent() ? owner->getParent()->getSubsystem() : nullptr;
  if (subsystem) {
    const uint64_t checkpointStart = diagnosticSample ? ActivityDiagnostics::timestamp() : 0;
    const bool terminal =
        subsystem->userReturnCheckpoint(*owner, frame) == Subsystem::UserReturnResult::Terminal;
    if (diagnosticSample) {
      ActivityDiagnostics::recordUserReturnStage(
          ActivityDiagnostics::UserReturnStage::Checkpoint,
          ActivityDiagnostics::timestamp() - checkpointStart);
    }
    if (terminal)
      return finishWork(true);
  }
 
  // Terminal requests and process stops win over later work. The architecture
  // caller owns the final commit after its return-tail scopes and accounting
  // have retired.
  uint64_t stageStart = diagnosticSample ? ActivityDiagnostics::timestamp() : 0;
  bool terminal = Processor::information().getScheduler().serviceProcessStopAtUserReturn();
  if (diagnosticSample) {
    ActivityDiagnostics::recordUserReturnStage(ActivityDiagnostics::UserReturnStage::ProcessStop,
                                               ActivityDiagnostics::timestamp() - stageStart);
  }
  if (terminal)
    return finishWork(true);
 
  Processor::information().getScheduler().serviceDeferredSubsystemException(state,
                                                                            diagnosticSample);
  Thread* current = Processor::information().getCurrentThread();
  if (current && !current->isTerminationDeferred() &&
      current->getUnwindState() != Thread::Continue) {
    return finishWork(true);
  }
 
#if PEDIGREE_FAST_USER_RETURN
  // Resolving the deferred fault may have retired the only pending work.
  if ((origin == UserReturnFrame::Origin::Syscall ||
       origin == UserReturnFrame::Origin::Interrupt) &&
      !frame.m_Terminal && owner->canSkipUserReturnWork()) {
    return finishWork(false);
  }
#endif
 
  stageStart = diagnosticSample ? ActivityDiagnostics::timestamp() : 0;
  terminal = Processor::information().getScheduler().serviceProcessStopAtUserReturn();
  if (diagnosticSample) {
    ActivityDiagnostics::recordUserReturnStage(ActivityDiagnostics::UserReturnStage::ProcessStop,
                                               ActivityDiagnostics::timestamp() - stageStart);
  }
  if (terminal)
    return finishWork(true);
 
  stageStart = diagnosticSample ? ActivityDiagnostics::timestamp() : 0;
  Processor::information().getScheduler().checkEventState(
      state.getStackPointer(), Thread::EventSelection::AnyDeliverable, &state, nullptr);
  if (diagnosticSample) {
    ActivityDiagnostics::recordUserReturnStage(ActivityDiagnostics::UserReturnStage::Event,
                                               ActivityDiagnostics::timestamp() - stageStart);
  }
 
  stageStart = diagnosticSample ? ActivityDiagnostics::timestamp() : 0;
  terminal =
      frame.m_Terminal || Processor::information().getScheduler().serviceProcessStopAtUserReturn();
  if (diagnosticSample) {
    ActivityDiagnostics::recordUserReturnStage(ActivityDiagnostics::UserReturnStage::ProcessStop,
                                               ActivityDiagnostics::timestamp() - stageStart);
  }
  if (!terminal)
    owner->clearUserReturnWorkIfIdle();
  return finishWork(terminal);
}
 
bool PerProcessorScheduler::serviceUserReturnWork(SyscallState& state,
                                                  UserReturnFrame::Origin origin,
                                                  bool diagnosticSample) {
  const uint64_t workStart = diagnosticSample ? ActivityDiagnostics::timestamp() : 0;
  auto finishWork = [diagnosticSample, workStart](bool terminal) {
    if (diagnosticSample) {
      ActivityDiagnostics::recordUserReturnStage(ActivityDiagnostics::UserReturnStage::SyscallWork,
                                                 ActivityDiagnostics::timestamp() - workStart);
    }
    return terminal;
  };
  Thread* owner = Processor::information().getCurrentThread();
  if (!owner)
    return finishWork(false);
#if X64 && !HOSTED
  {
    EnsureInterrupts interrupts(false);
    state.setFlags(state.getFlags() | 0x202);
  }
#endif
#if PEDIGREE_FAST_USER_RETURN
  if (origin == UserReturnFrame::Origin::Syscall && owner->canSkipUserReturnWork())
    return finishWork(false);
#endif
 
  UserReturnFrame frame(*owner, state, origin);
  Thread::UserReturnFrameScope frameScope(*owner, frame);
  Subsystem* subsystem = owner->getParent() ? owner->getParent()->getSubsystem() : nullptr;
  if (subsystem) {
    const uint64_t checkpointStart = diagnosticSample ? ActivityDiagnostics::timestamp() : 0;
    const bool terminal =
        subsystem->userReturnCheckpoint(*owner, frame) == Subsystem::UserReturnResult::Terminal;
    if (diagnosticSample) {
      ActivityDiagnostics::recordUserReturnStage(
          ActivityDiagnostics::UserReturnStage::Checkpoint,
          ActivityDiagnostics::timestamp() - checkpointStart);
    }
    if (terminal)
      return finishWork(true);
  }
  uint64_t stageStart = diagnosticSample ? ActivityDiagnostics::timestamp() : 0;
  bool terminal = Processor::information().getScheduler().serviceProcessStopAtUserReturn();
  if (diagnosticSample) {
    ActivityDiagnostics::recordUserReturnStage(ActivityDiagnostics::UserReturnStage::ProcessStop,
                                               ActivityDiagnostics::timestamp() - stageStart);
  }
  if (terminal)
    return finishWork(true);
  stageStart = diagnosticSample ? ActivityDiagnostics::timestamp() : 0;
  Processor::information().getScheduler().checkEventState(
      state.getStackPointer(), Thread::EventSelection::AnyDeliverable, nullptr, &state);
  if (diagnosticSample) {
    ActivityDiagnostics::recordUserReturnStage(ActivityDiagnostics::UserReturnStage::Event,
                                               ActivityDiagnostics::timestamp() - stageStart);
  }
  stageStart = diagnosticSample ? ActivityDiagnostics::timestamp() : 0;
  terminal =
      frame.m_Terminal || Processor::information().getScheduler().serviceProcessStopAtUserReturn();
  if (diagnosticSample) {
    ActivityDiagnostics::recordUserReturnStage(ActivityDiagnostics::UserReturnStage::ProcessStop,
                                               ActivityDiagnostics::timestamp() - stageStart);
  }
  if (!terminal)
    owner->clearUserReturnWorkIfIdle();
  return finishWork(terminal);
}
 
void PerProcessorScheduler::serviceDeferredSubsystemException(InterruptState& state,
                                                              bool diagnosticSample) {
  Thread* thread = Processor::information().getCurrentThread();
  if (!thread) {
    return;
  }
 
  size_t rawType = 0;
  uintptr_t faultAddress = 0;
  uintptr_t errorCode = 0;
  if (!thread->takeDeferredSubsystemException(rawType, faultAddress, errorCode)) {
    return;
  }
 
  Process* process = thread->getParent();
  Subsystem* subsystem = process ? process->getSubsystem() : nullptr;
  if (!subsystem || rawType > static_cast<size_t>(Subsystem::Other)) {
    FATAL("Deferred userspace exception has no valid owning subsystem");
  }
 
  // Disk-backed faults can only wait once the raw interrupt and accounting
  // scopes are gone. An unsuccessful retry retains ordinary signal delivery.
  if (rawType == static_cast<size_t>(Subsystem::PageFault) && !state.kernelMode() &&
      Processor::getInterrupts()) {
    const uint64_t faultStart = diagnosticSample ? ActivityDiagnostics::timestamp() : 0;
    const bool handled = subsystem->resolveUserPageFault(*thread, state, faultAddress, errorCode);
    if (diagnosticSample) {
      ActivityDiagnostics::recordUserReturnStage(
          ActivityDiagnostics::UserReturnStage::DeferredFault,
          ActivityDiagnostics::timestamp() - faultStart);
      ActivityDiagnostics::recordUserReturnFaultOutcome(handled);
    }
    if (handled)
      return;
  }
 
  subsystem->threadException(thread, static_cast<Subsystem::ExceptionType>(rawType), &state,
                             faultAddress, errorCode);
}
 
void PerProcessorScheduler::commitUserReturnTerminalState() {
  Thread* thread = Processor::information().getCurrentThread();
  if (!thread || thread->isTerminationDeferred()) {
    return;
  }
 
  const Thread::UnwindType unwindState = thread->getUnwindState();
  if (unwindState == Thread::TerminateThread) {
    commitCurrentThreadExit();
  }
  if (unwindState == Thread::Exit) {
    Process* process = thread->getParent();
    Subsystem* subsystem = process ? process->getSubsystem() : nullptr;
    if (!subsystem) {
      FATAL("Return-to-user process exit has no owning subsystem.");
    }
    const Thread::DeferredProcessExit request = thread->takeDeferredProcessExit();
    subsystem->exit(request.code, request.cause);
    FATAL("Subsystem::exit returned to a user-return boundary.");
  }
}
 
#if HOSTED && PEDIGREE_HOSTED_SMOKE_TESTS
namespace {
struct HostedRunnableCurrentContext {
  HostedRunnableCurrentContext(Thread* current, Thread* idleOwner)
      : driver(current), idle(idleOwner), idleWhileRunnable(0) {}
 
  Thread* driver;
  Thread* idle;
  Atomic<size_t> idleWhileRunnable;
};
 
HostedRunnableCurrentContext* g_HostedRunnableCurrentContext = nullptr;
 
void observeHostedRunnableCurrent(ProcessorBase::HostedContextSwitchStage stage) {
  auto* context = __atomic_load_n(&g_HostedRunnableCurrentContext, __ATOMIC_ACQUIRE);
  if (context && stage == ProcessorBase::HostedContextSwitchStage::SwitchStateReturnedMasked &&
      Processor::information().getCurrentThread() == context->idle &&
      context->driver->getStatus() == Thread::Ready) {
    context->idleWhileRunnable += 1;
  }
}
 
struct HostedNewThreadContext {
  Atomic<size_t> calls;
};
 
int hostedNewThreadWorkerEntry(void* parameter) {
  HostedNewThreadContext* context = reinterpret_cast<HostedNewThreadContext*>(parameter);
  context->calls += 1;
  return 0;
}
 
}  // namespace
 
bool PerProcessorScheduler::currentIrqWorkDoorbellPendingForTest() {
  return Processor::information().getScheduler().m_IrqWorkDoorbell.value() != 0;
}
 
void PerProcessorScheduler::serviceCurrentIrqWorkDoorbellForTest() {
  Processor::information().getScheduler().serviceIrqWorkDoorbell();
}
 
bool PerProcessorScheduler::runHostedNewThreadWorkerRegressions() {
  if (this != &Processor::information().getScheduler() || !m_NewThreadWorker) {
    ERROR(
        "HOSTED-WAIT-TEST: per-processor worker regression requires "
        "the active scheduler");
    return false;
  }
 
  constexpr size_t Attempts = 10000;
  auto isParked = [this](Thread* target) {
    bool found = false;
    m_NewThreadDataLock.acquire();
    for (List<void*>::Iterator it = m_DelayedNewThreadData.begin();
         it != m_DelayedNewThreadData.end(); ++it) {
      newThreadData* pData = reinterpret_cast<newThreadData*>(*it);
      if (pData->pThread == target) {
        found = true;
        break;
      }
    }
    m_NewThreadDataLock.release();
    return found;
  };
  auto waitUntilParked = [&isParked, Attempts](Thread* target) {
    for (size_t attempt = 0; attempt < Attempts; ++attempt) {
      if (isParked(target)) {
        return true;
      }
      Scheduler::instance().yield();
    }
    return false;
  };
  auto check = [](bool condition, const char* message) {
    if (!condition) {
      ERROR("HOSTED-WAIT-TEST: " << message);
    }
    return condition;
  };
 
  Process* kernelProcess = Processor::information().getCurrentThread()->getParent();
  bool passed = true;
 
  HostedRunnableCurrentContext runnableContext(Processor::information().getCurrentThread(),
                                               m_pIdleThread);
  if (!check(runnableContext.idle && runnableContext.idle != runnableContext.driver,
             "runnable-current regression requires an ordinary thread and an idle owner")) {
    return false;
  }
  __atomic_store_n(&g_HostedRunnableCurrentContext, &runnableContext, __ATOMIC_RELEASE);
  Processor::setHostedContextSwitchHook(observeHostedRunnableCurrent);
  for (size_t attempt = 0; attempt < 256; ++attempt) {
    Scheduler::instance().yield();
  }
  Processor::setHostedContextSwitchHook(nullptr);
  __atomic_store_n(&g_HostedRunnableCurrentContext,
                   static_cast<HostedRunnableCurrentContext*>(nullptr), __ATOMIC_RELEASE);
  const bool runnablePassed =
      check(!runnableContext.idleWhileRunnable && Processor::getInterrupts() &&
                Processor::information().getCurrentThread() == runnableContext.driver &&
                runnableContext.driver->getStatus() == Thread::Running,
            "scheduler entered idle while the yielding thread remained runnable");
  passed &= runnablePassed;
  if (runnablePassed) {
    NOTICE("HOSTED-WAIT-TEST: PASS scheduler-runnable-current-keeps-cpu");
  }
 
  HostedNewThreadContext reapContext;
  const size_t reapBaseline = m_nDeferredThreadReapCompletions.value();
  Thread* reapTarget = new Thread(kernelProcess, hostedNewThreadWorkerEntry, &reapContext, nullptr,
                                  false, true, true);
  reapTarget->setName("hosted deferred Thread reap target");
  const bool reapStarted = reapTarget->startDetached();
  bool reapCompleted = false;
  for (size_t attempt = 0; attempt < Attempts; ++attempt) {
    if (m_nDeferredThreadReapCompletions.value() == reapBaseline + 1) {
      reapCompleted = true;
      break;
    }
    Scheduler::instance().yield();
  }
  const bool reapPassed = check(
      reapStarted && reapCompleted && reapContext.calls == 1 && !m_nDeferredThreadReaps.value(),
      "detached Thread was not destroyed by the ordinary maintenance worker");
  passed &= reapPassed;
  if (reapPassed) {
    NOTICE("HOSTED-WAIT-TEST: PASS perprocessor-deferred-thread-reap");
  }
 
  HostedNewThreadContext delayedContext;
  Thread* delayed = new Thread(kernelProcess, hostedNewThreadWorkerEntry, &delayedContext, nullptr,
                               false, true, true);
  delayed->setName("hosted delayed add-worker target");
  const bool delayedParked = waitUntilParked(delayed);
  for (size_t attempt = 0; attempt < 64; ++attempt) {
    Scheduler::instance().yield();
  }
  const bool stayedDormant = delayedParked && isParked(delayed) && delayedContext.calls == 0;
  const bool started = delayed->start();
  const bool delayedJoined = delayed->joinForCompletion();
  const bool delayedPassed =
      check(stayedDormant && started && delayedJoined && delayedContext.calls == 1,
            "delayed add-worker target did not remain parked until its single "
            "start publication");
  passed &= delayedPassed;
  if (delayedPassed) {
    NOTICE("HOSTED-WAIT-TEST: PASS perprocessor-delayed-start-wake");
  }
 
  HostedNewThreadContext terminatedContext;
  Thread* terminated = new Thread(kernelProcess, hostedNewThreadWorkerEntry, &terminatedContext,
                                  nullptr, false, true, true);
  terminated->setName("hosted terminated add-worker target");
  const bool terminatedParked = waitUntilParked(terminated);
  terminated->setUnwindState(Thread::TerminateThread);
  const bool terminatedJoined = terminated->joinForCompletion();
  const bool terminatedPassed =
      check(terminatedParked && terminatedJoined && terminatedContext.calls == 0,
            "terminate-before-start did not retire the parked add-worker target");
  passed &= terminatedPassed;
  if (terminatedPassed) {
    NOTICE("HOSTED-WAIT-TEST: PASS perprocessor-terminate-before-start");
  }
 
  HostedNewThreadContext teardownContext;
  Thread* teardown = new Thread(kernelProcess, hostedNewThreadWorkerEntry, &teardownContext,
                                nullptr, false, true, true);
  teardown->setName("hosted add-worker teardown target");
  const bool teardownParked = waitUntilParked(teardown);
 
  HostedNewThreadContext detachedTeardownContext;
  const size_t detachedReapBaseline = m_nDeferredThreadReapCompletions.value();
  Thread* detachedTeardown = new Thread(kernelProcess, hostedNewThreadWorkerEntry,
                                        &detachedTeardownContext, nullptr, false, true, true);
  detachedTeardown->setName("hosted detached add-worker teardown target");
  const bool detachedTeardownParked = waitUntilParked(detachedTeardown);
  const bool detachedTeardownClaimed = detachedTeardown->detach();
 
  stopNewThreadWorker();
  const bool teardownJoined = teardown->joinForCompletion();
  bool detachedTeardownReaped = false;
  for (size_t attempt = 0; attempt < Attempts; ++attempt) {
    if (m_nDeferredThreadReapCompletions.value() == detachedReapBaseline + 1) {
      detachedTeardownReaped = true;
      break;
    }
    Scheduler::instance().yield();
  }
 
  m_NewThreadDataLock.acquire();
  const bool teardownDrained = !m_NewThreadAdmissionOpen && m_StopNewThreadWorker &&
                               !m_NewThreadData.count() && !m_DelayedNewThreadData.count();
  m_NewThreadDataLock.release();
  const bool workerJoined = !m_NewThreadWorker;
 
  // A joined worker cannot be hiding on the condition variable or retain a
  // detached reference to this scheduler's queue state.
  m_NewThreadDataCondition.broadcast();
  for (size_t attempt = 0; attempt < 64; ++attempt) {
    Scheduler::instance().yield();
  }
 
  const bool teardownPassed = check(
      teardownParked && teardownJoined && teardownContext.calls == 0 && detachedTeardownParked &&
          detachedTeardownClaimed && detachedTeardownReaped && detachedTeardownContext.calls == 0 &&
          !m_nDeferredThreadReaps.value() && teardownDrained && workerJoined,
      "owned add worker did not drain and join with pending parked work");
  passed &= teardownPassed;
  if (teardownPassed) {
    NOTICE("HOSTED-WAIT-TEST: PASS perprocessor-worker-teardown");
  }
 
  startNewThreadWorker(kernelProcess);
 
  HostedNewThreadContext restartContext;
  Thread* restart = new Thread(kernelProcess, hostedNewThreadWorkerEntry, &restartContext, nullptr,
                               false, true, true);
  restart->setName("hosted restarted add-worker target");
  const bool restartParked = waitUntilParked(restart);
  const bool restartStarted = restart->start();
  bool restartReapable = false;
  for (size_t attempt = 0; attempt < Attempts; ++attempt) {
    if (restart->isReapableForHostedTest()) {
      restartReapable = true;
      break;
    }
    Scheduler::instance().yield();
  }
  const bool restartJoined = restartReapable && restart->joinForCompletion();
  const bool restartPassed =
      check(restartParked && restartStarted && restartJoined && restartContext.calls == 1,
            "replacement add worker did not process a fresh delayed admission");
  passed &= restartPassed;
  if (restartPassed) {
    NOTICE("HOSTED-WAIT-TEST: PASS perprocessor-worker-restart");
  }
 
  return passed;
}
#endif
 
void PerProcessorScheduler::requestIdleThreadWakeup() {
  __atomic_store_n(&m_IdleWakeRequested, true, __ATOMIC_RELEASE);
  prompt();
}
 
void PerProcessorScheduler::setIdle(Thread* pThread) {
  __atomic_store_n(&m_pIdleThread, pThread, __ATOMIC_RELEASE);
  if (!pThread)
    __atomic_store_n(&m_IdleWakeRequested, false, __ATOMIC_RELEASE);
}
 
void PerProcessorScheduler::idleUntilInterrupt() {
  Processor::setInterrupts(false);
  servicePendingScheduling();
  updateOneShotTimer();
  if (__atomic_load_n(&m_IdleWakeRequested, __ATOMIC_ACQUIRE)) {
    Processor::setInterrupts(true);
    return;
  }
  if (m_pSchedulingAlgorithm && m_pSchedulingAlgorithm->hasReady()) {
    schedule(Thread::Ready, false);
  } else {
    // The ready check and STI/HLT are one interrupt-masked handshake. A
    // remote IPI arriving after the check remains pending until STI/HLT.
    Processor::haltUntilInterrupt();
  }
  Processor::setInterrupts(true);
}