Inheritance diagram for PerProcessorScheduler:

Collaboration diagram for PerProcessorScheduler:

Public Types
enum class	StackDiscardReason { EmergencyProcessKill , HostedRegression , LegacyAbiCall }

enum class	ProcessStopGateMode { StopOnly , DirectUserTransition }

Public Member Functions
	PerProcessorScheduler ()

void	initialise (Thread *pThread)

void	checkEventState (uintptr_t userStack)

void	eventHandlerReturned () NORETURN

void	addThread (Thread pThread, Thread::ThreadStartFunc pStartFunction, void pParam, bool bUsermode, void *pStack)

void	addThread (Thread *pThread, SyscallState &state)

void	commitCurrentThreadExit (Spinlock *pLock=0) NORETURN

void	abandonCurrentThreadStack (StackDiscardReason reason, Spinlock *pLock=0) NORETURN

void	requestCurrentThreadExitToIdle ()

void	timer (uint64_t delta, InterruptState &state)

void	setClockDeadline (uint64_t deadline)

void	removeThread (Thread *pThread)

void	threadStatusChanged (Thread *pThread)

size_t	logicalCpu () const

bool	isIdleThread (const Thread *thread) const

uint64_t	nominalQuantumNs () const

void	ringIrqWorkDoorbell ()

void	registerWorkerWake (SchedulerWorkerWake &worker, WaitQueue &waiters)

void	unregisterWorkerWake (SchedulerWorkerWake &worker)

void	ringIrqWorkDoorbell (SchedulerWorkerWake &worker, bool promptOwner=true)

void	publishDeferredTimeAccounting ()

void	serviceIrqWorkDoorbell ()

void	servicePendingScheduling ()

MUST_USE_RESULT bool	serviceProcessStopAtUserReturn (ProcessStopGateMode mode=ProcessStopGateMode::StopOnly)

MUST_USE_RESULT bool	serviceUserReturnWork (InterruptState &state, UserReturnFrame::Origin origin=UserReturnFrame::Origin::Interrupt, bool diagnosticSample=false)

MUST_USE_RESULT bool	serviceUserReturnWork (SyscallState &state, UserReturnFrame::Origin origin=UserReturnFrame::Origin::Syscall, bool diagnosticSample=false)

bool	sampleUserReturnDiagnostics ()

void	commitUserReturnTerminalState ()

void	requestIdleThreadWakeup ()

void	setIdle (Thread *pThread)

void	idleUntilInterrupt ()

Static Public Member Functions
static size_t	stackDiscardCount ()

static size_t	stackDiscardCount (StackDiscardReason reason)

Private Member Functions
void	killCurrentThread (Spinlock *pLock=0) NORETURN

void	schedule (Thread::Status nextStatus=Thread::Ready, bool dispatchEvents=true)

void	scheduleWithInterruptState (Thread::Status nextStatus, bool dispatchEvents, bool previousInterruptState)

void	blockCurrent ()

void	publishReadyFromWait (Thread *pThread)

void	finishCurrentThreadExit (Spinlock *pLock, bool transferToIdle) NORETURN

void	serviceDeferredSubsystemException (InterruptState &state, bool diagnosticSample=false)

void	checkEventState (uintptr_t userStack, Thread::EventSelection selection)

void	checkEventState (uintptr_t userStack, Thread::EventSelection selection, InterruptState interruptState, SyscallState syscallState)

	PerProcessorScheduler (const PerProcessorScheduler &)

PerProcessorScheduler &	operator= (const PerProcessorScheduler &)

void	startNewThreadWorker (Process *pParent)

void	stopNewThreadWorker ()

void	startTimeAccountingWorker (Process *pParent)

void	stopTimeAccountingWorker ()

int	runTimeAccountingWorker ()

void	publishDeferredThreadReap (Thread *thread)

bool	drainDeferredThreadReaps ()

bool	enqueueAffinity (Thread *thread, bool accepted=false)

void	drainAffinityRequests ()

void	prompt (bool readyPublication=false)

void	armLocalQuantumIfNeeded ()

void	updateOneShotTimer ()

void	programOneShotTimer ()

Thread *	selectNext (Thread *current)

void	serviceWorkerWakeups ()

Static Private Member Functions
static void	deleteThreadThenRestoreState (Thread pThread, SchedulerState &newState, volatile uintptr_t pLock=0) NORETURN

static void	deleteThread (Thread *pThread)

static int	timeAccountingWorkerEntry (void *instance)

static int	processorAddThread (void *instance)

Private Attributes
SchedulingAlgorithm *	m_pSchedulingAlgorithm

Mutex	m_NewThreadDataLock

ConditionVariable	m_NewThreadDataCondition

List< void * >	m_NewThreadData

List< void * >	m_DelayedNewThreadData

bool	m_NewThreadAdmissionOpen

bool	m_StopNewThreadWorker

OwnedThread	m_NewThreadWorker

DeferredTimeAccountingWorkerState	m_TimeAccountingState

DeferredThreadReapNode	m_DeferredThreadReapStub

IntrusiveMpscQueue< DeferredThreadReapNode, &DeferredThreadReapNode::next >	m_DeferredThreadReaps

Atomic< size_t >	m_nDeferredThreadReaps

Atomic< size_t >	m_DeferredThreadReapPublicationState

Atomic< size_t >	m_StopTimeAccountingWorker

OwnedThread	m_TimeAccountingWorker

WaitQueue	m_TimeAccountingWorkerWaiters

SchedulerWorkerWake	m_TimeAccountingWorkerWake

Atomic< size_t >	m_IrqWorkDoorbell

Atomic< size_t >	m_ReschedulePending

Atomic< size_t >	m_RemotePromptPending

Atomic< uint64_t >	m_ClockDeadline

Spinlock	m_IrqWorkLock

SchedulerWorkerWake *	m_pWorkerWakeHead = nullptr

Spinlock	m_AffinityQueueLock

Thread *	m_AffinityHead = nullptr

Thread *	m_AffinityTail = nullptr

Atomic< size_t >	m_AffinityRequests

bool	m_AffinityAdmissionOpen = false

size_t	m_LogicalCpu = ~size_t(0)

size_t	m_PhysicalCpu = 0

uint64_t	m_NominalQuantumNs = 0

uint64_t	m_QuantumDeadline = 0

uint64_t	m_NextLoadSampleDeadline = 0

bool	m_OneShotTimer = false

Thread *	m_pIdleThread

bool	m_IdleWakeRequested = false

size_t	m_SchedulerTickCounter = 0

size_t	m_UserReturnDiagnosticSequence = 0

Static Private Attributes
static constexpr size_t	DeferredReapPublicationClosed

static constexpr size_t	DeferredReapPublicationCountMask = ~DeferredReapPublicationClosed

Friends
class	Scheduler

class	Thread

class	ProcessorThreadAllocator

class	WaitQueue

Detailed Description

Definition at line 45 of file PerProcessorScheduler.h.

Member Enumeration Documentation

◆ ProcessStopGateMode

enum class PerProcessorScheduler::ProcessStopGateMode

strong

Definition at line 155 of file PerProcessorScheduler.h.

◆ StackDiscardReason

enum class PerProcessorScheduler::StackDiscardReason

strong

Why an exceptional exit is intentionally discarding live C++ frames.

Definition at line 48 of file PerProcessorScheduler.h.

Constructor & Destructor Documentation

◆ PerProcessorScheduler() [1/2]

PerProcessorScheduler::PerProcessorScheduler ( )

Default constructor - Creates an empty scheduler with a new idle thread.

Definition at line 69 of file PerProcessorScheduler.cc.

◆ ~PerProcessorScheduler()

PerProcessorScheduler::~PerProcessorScheduler ( )

Definition at line 98 of file PerProcessorScheduler.cc.

◆ PerProcessorScheduler() [2/2]

PerProcessorScheduler::PerProcessorScheduler ( const PerProcessorScheduler & )

private

Copy-constructor

Note: Not implemented - singleton class.

Member Function Documentation

◆ abandonCurrentThreadStack()

void PerProcessorScheduler::abandonCurrentThreadStack	(	StackDiscardReason	reason,
		Spinlock *	pLock = `0`
	)

Explicitly discards the current physical stack for exceptional recovery. Every call is counted and stack-owned cleanup records are retired on a best-effort basis before switching away.

Definition at line 1270 of file PerProcessorScheduler.cc.

References ProcessorBase::information(), Thread::m_ExitToIdle, and Thread::unlinkWaitsForStackDiscard().

Referenced by killCurrentThread().

Here is the caller graph for this function:

◆ addThread() [1/2]

void PerProcessorScheduler::addThread	(	Thread *	pThread,
		SyscallState &	state
	)

Adds a new thread.

Parameters

pThread	The thread to add.
state	The syscall state to jump to.

Definition at line 1129 of file PerProcessorScheduler.cc.

References Spinlock::acquire(), Semaphore::acquire(), SchedulingAlgorithm::addThread(), LocksCommand::checkSchedule(), Spinlock::deferredReleaseWord(), EMIT_IF, Process::getAddressSpace(), ProcessorBase::getInterrupts(), Thread::getKernelStack(), Thread::getLock(), Thread::getParent(), Thread::getStatus(), Thread::getTlsBase(), ProcessorBase::id(), ProcessorBase::information(), LocksCommand::lockReleased(), Thread::m_Lock, m_pSchedulingAlgorithm, panic(), List< T, nodePoolSize >::pushBack(), Thread::recordTime(), Spinlock::release(), Semaphore::release(), ProcessorBase::restoreState(), ProcessorBase::saveState(), Thread::setCpuId(), ProcessorBase::setInterrupts(), Thread::setScheduler(), ProcessorBase::setTlsBase(), ConditionVariable::signal(), Thread::state(), ProcessorBase::switchAddressSpace(), ProcessorBase::switchState(), and Thread::trackTime().

◆ addThread() [2/2]

void PerProcessorScheduler::addThread	(	Thread *	pThread,
		Thread::ThreadStartFunc	pStartFunction,
		void *	pParam,
		bool	bUsermode,
		void *	pStack
	)

Adds a new thread.

Parameters

pThread	The thread to add.
pStartFunction	The function to start the thread with.
pParam	void* parameter to give to the function.
bUsermode	Start the thread in User Mode?
pStack	Stack to start the thread with.

Definition at line 1010 of file PerProcessorScheduler.cc.

References Spinlock::acquire(), Semaphore::acquire(), SchedulingAlgorithm::addThread(), LocksCommand::checkSchedule(), Spinlock::deferredReleaseWord(), EMIT_IF, Process::getAddressSpace(), ProcessorBase::getInterrupts(), Thread::getKernelStack(), Thread::getLock(), Thread::getParent(), Thread::getStatus(), Thread::getTlsBase(), ProcessorBase::id(), ProcessorBase::information(), ProcessorBase::jumpKernel(), ProcessorBase::jumpUser(), LocksCommand::lockReleased(), Thread::m_Lock, m_pSchedulingAlgorithm, panic(), List< T, nodePoolSize >::pushBack(), Thread::recordTime(), Spinlock::release(), Semaphore::release(), ProcessorBase::saveAndJumpKernel(), ProcessorBase::saveAndJumpUser(), ProcessorBase::saveState(), Thread::setCpuId(), ProcessorBase::setInterrupts(), Thread::setScheduler(), ProcessorBase::setTlsBase(), ConditionVariable::signal(), Thread::state(), ProcessorBase::switchAddressSpace(), and Thread::trackTime().

Referenced by ProcessorThreadAllocator::addThread(), and ProcessorThreadAllocator::addThread().

Here is the caller graph for this function:

◆ armLocalQuantumIfNeeded()

void PerProcessorScheduler::armLocalQuantumIfNeeded ( )

private

Definition at line 527 of file PerProcessorScheduler.cc.

◆ blockCurrent()

void PerProcessorScheduler::blockCurrent ( )

private

Blocks the current thread after WaitQueue has published its wait record.

Definition at line 1464 of file PerProcessorScheduler.cc.

References schedule().

◆ checkEventState() [1/3]

void PerProcessorScheduler::checkEventState ( uintptr_t userStack )

Looks for event handlers to run, and if found, dispatches one.

Parameters

userStack The stack to use if the event has a user-mode handler. Usually obtained from an interruptState or syscallState.

Definition at line 740 of file PerProcessorScheduler.cc.

References checkEventState().

Referenced by checkEventState(), checkEventState(), and PosixSubsystem::sendSignal().

Here is the caller graph for this function:

◆ checkEventState() [2/3]

void PerProcessorScheduler::checkEventState	(	uintptr_t	userStack,
		Thread::EventSelection	selection
	)

private

Dispatches one event using an exact scheduler-owned selection policy.

Definition at line 744 of file PerProcessorScheduler.cc.

References checkEventState().

◆ checkEventState() [3/3]

void PerProcessorScheduler::checkEventState	(	uintptr_t	userStack,
		Thread::EventSelection	selection,
		InterruptState *	interruptState,
		SyscallState *	syscallState
	)

private

Dispatches one event with an optional raw return-to-user context.

Definition at line 748 of file PerProcessorScheduler.cc.

References PhysicalMemoryManager::allocatePage(), VirtualAddressSpace::allocateStack(), Event::deliverAtUserReturn(), EMIT_IF, Thread::eventsDeferred(), Event::getHandlerAddress(), Event::getHandlerBuffer(), Event::getHandlerBufferSize(), Event::getHandlerPrivilege(), Thread::getId(), ProcessorBase::getInterrupts(), Thread::getKernelStack(), VirtualAddressSpace::getKernelStart(), VirtualAddressSpace::getMapping(), Thread::getNextEvent(), Event::getNumber(), PhysicalMemoryManager::getPageSize(), Thread::getParent(), Thread::getScheduler(), Thread::getStateLevel(), Event::getTrampoline(), VirtualAddressSpace::getUserStart(), Hex, ProcessorBase::information(), PhysicalMemoryManager::instance(), VirtualAddressSpace::isAddressValid(), Event::isDeletable(), VirtualAddressSpace::isMapped(), Event::isSignalEvent(), Event::isValidHandlerMapping(), ProcessorBase::jumpUser(), VirtualAddressSpace::KernelMode, Thread::m_AlternateSignalStack, Thread::StateLevel::m_bOwnsAlternateSignalStack, Thread::m_nStateLevel, VirtualAddressSpace::map(), Thread::markDeferredUserReturnSignalInterruption(), panic(), Thread::popState(), Event::prefersAlternateUserStack(), Thread::pushState(), Event::requiresExactUserReturnState(), Event::Delivery::reset(), VirtualAddressSpace::RuntimeMapping, ProcessorBase::saveAndJumpUser(), ProcessorBase::saveState(), Thread::sendEvent(), Event::serialize(), ProcessorBase::setInterrupts(), Thread::transitionTime(), and VirtualAddressSpace::Write.

◆ commitCurrentThreadExit()

void PerProcessorScheduler::commitCurrentThreadExit ( Spinlock * pLock = 0 )

Commits an exit after the current C++ call chain has returned to an audited boundary.

This refuses to switch away while stack-owned cleanup records, deferral scopes, or active WaitQueue records remain.

Definition at line 1241 of file PerProcessorScheduler.cc.

References ProcessorBase::information(), Thread::m_EventDeferralDepth, Thread::m_ExitToIdle, Thread::m_pDeferredScopes, and Thread::m_TerminationDeferralDepth.

Referenced by commitUserReturnTerminalState().

Here is the caller graph for this function:

◆ commitUserReturnTerminalState()

void PerProcessorScheduler::commitUserReturnTerminalState ( )

Commits terminal state after architecture return-tail cleanup is done.

Definition at line 1981 of file PerProcessorScheduler.cc.

References commitCurrentThreadExit(), Thread::Exit, Subsystem::exit(), Thread::getParent(), Thread::getUnwindState(), ProcessorBase::information(), Thread::isTerminationDeferred(), Thread::takeDeferredProcessExit(), and Thread::TerminateThread.

◆ deleteThread()

void PerProcessorScheduler::deleteThread ( Thread * pThread )

staticprivate

Definition at line 1392 of file PerProcessorScheduler.cc.

◆ deleteThreadThenRestoreState()

static void PerProcessorScheduler::deleteThreadThenRestoreState	(	Thread *	pThread,
		SchedulerState &	newState,
		volatile uintptr_t *	pLock = `0`
	)

staticprivate

Switches stacks, calls PerProcessorScheduler::deleteThread, then context switches.

Note: Implemented in core/processor/ARCH/asm

◆ drainAffinityRequests()

void PerProcessorScheduler::drainAffinityRequests ( )

private

Definition at line 296 of file Thread-affinity.cc.

◆ drainDeferredThreadReaps()

bool PerProcessorScheduler::drainDeferredThreadReaps ( )

private

Definition at line 228 of file PerProcessorScheduler.cc.

◆ enqueueAffinity()

bool PerProcessorScheduler::enqueueAffinity	(	Thread *	thread,
		bool	accepted = `false`
	)

private

Definition at line 262 of file Thread-affinity.cc.

◆ eventHandlerReturned()

void PerProcessorScheduler::eventHandlerReturned ( )

Assumes this thread has just returned from executing a event handler, and lets it resume normal execution.

Definition at line 1000 of file PerProcessorScheduler.cc.

References Thread::abandonCurrentState(), ProcessorBase::information(), ProcessorBase::restoreState(), ProcessorBase::setInterrupts(), and Thread::state().

Referenced by HostedSyscallManager::syscall().

Here is the caller graph for this function:

◆ finishCurrentThreadExit()

void PerProcessorScheduler::finishCurrentThreadExit	(	Spinlock *	pLock,
		bool	transferToIdle
	)

private

Definition at line 1326 of file PerProcessorScheduler.cc.

◆ idleUntilInterrupt()

void PerProcessorScheduler::idleUntilInterrupt ( )

Checks ready work with IRQs masked before the idle thread halts.

Definition at line 2256 of file PerProcessorScheduler.cc.

References ProcessorBase::haltUntilInterrupt(), SchedulingAlgorithm::hasReady(), m_pSchedulingAlgorithm, schedule(), servicePendingScheduling(), and ProcessorBase::setInterrupts().

◆ initialise()

void PerProcessorScheduler::initialise ( Thread * pThread )

Initialises the scheduler with the given thread.

Definition at line 406 of file PerProcessorScheduler.cc.

References SchedulingAlgorithm::addThread(), ProcessorBase::getCount(), Thread::getKernelStack(), Thread::getParent(), Machine::getSchedulerTimer(), Thread::getTlsBase(), ProcessorBase::id(), ProcessorBase::index(), ProcessorBase::information(), ProcessorBase::informationAt(), m_pSchedulingAlgorithm, panic(), Thread::recordTime(), SchedulerTimer::registerHandler(), Thread::setCpuId(), Thread::setStatus(), ProcessorBase::setTlsBase(), and SchedulerTimer::supportsOneShot().

◆ isIdleThread()

bool PerProcessorScheduler::isIdleThread ( const Thread * thread ) const

inline

Definition at line 124 of file PerProcessorScheduler.h.

◆ killCurrentThread()

void PerProcessorScheduler::killCurrentThread ( Spinlock * pLock = 0 )

private

Link compatibility only; new source must choose an explicit exit API.

Definition at line 1266 of file PerProcessorScheduler.cc.

References abandonCurrentThreadStack().

◆ logicalCpu()

size_t PerProcessorScheduler::logicalCpu ( ) const

inline

Definition at line 121 of file PerProcessorScheduler.h.

◆ nominalQuantumNs()

uint64_t PerProcessorScheduler::nominalQuantumNs ( ) const

inline

Definition at line 127 of file PerProcessorScheduler.h.

◆ operator=()

PerProcessorScheduler & PerProcessorScheduler::operator= ( const PerProcessorScheduler & )

private

Assignment operator

Note: Not implemented - singleton class

◆ processorAddThread()

int PerProcessorScheduler::processorAddThread ( void * instance )

staticprivate

Definition at line 315 of file PerProcessorScheduler.cc.

◆ programOneShotTimer()

void PerProcessorScheduler::programOneShotTimer ( )

private

Definition at line 457 of file PerProcessorScheduler.cc.

◆ prompt()

void PerProcessorScheduler::prompt ( bool readyPublication = false )

private

Definition at line 277 of file Thread-affinity.cc.

◆ publishDeferredThreadReap()

void PerProcessorScheduler::publishDeferredThreadReap ( Thread * thread )

private

Definition at line 208 of file PerProcessorScheduler.cc.

◆ publishDeferredTimeAccounting()

void PerProcessorScheduler::publishDeferredTimeAccounting ( )

Publishes deferred process timer accounting from IRQ/scheduler context. The accounting worker is made runnable through the shared IRQ doorbell.

Definition at line 1654 of file PerProcessorScheduler.cc.

References DeferredTimeAccountingWorkerState::publish(), and ringIrqWorkDoorbell().

◆ publishReadyFromWait()

void PerProcessorScheduler::publishReadyFromWait ( Thread * pThread )

private

Publishes a completed wait directly to this scheduler's ready queue.

Definition at line 1468 of file PerProcessorScheduler.cc.

Referenced by Thread::sendEvent(), Thread::setUnwindState(), and Thread::wakeForDeliverableEvents().

Here is the caller graph for this function:

◆ registerWorkerWake()

void PerProcessorScheduler::registerWorkerWake	(	SchedulerWorkerWake &	worker,
		WaitQueue &	waiters
	)

Registers a worker wait queue for scheduler-side wake publication.

Definition at line 1596 of file PerProcessorScheduler.cc.

Referenced by Rtc::initialise3().

Here is the caller graph for this function:

◆ removeThread()

void PerProcessorScheduler::removeThread ( Thread * pThread )

Definition at line 1460 of file PerProcessorScheduler.cc.

◆ requestCurrentThreadExitToIdle()

void PerProcessorScheduler::requestCurrentThreadExitToIdle ( )

Selects the registered idle owner when the current Thread exits.

Definition at line 1318 of file PerProcessorScheduler.cc.

References ProcessorBase::information(), Thread::m_ExitToIdle, and panic().

◆ requestIdleThreadWakeup()

void PerProcessorScheduler::requestIdleThreadWakeup ( )

Wakes the idle owner until it retires its role for terminal shutdown.

Definition at line 2245 of file PerProcessorScheduler.cc.

◆ ringIrqWorkDoorbell() [1/2]

void PerProcessorScheduler::ringIrqWorkDoorbell ( )

Atomic hard-IRQ publication; does not touch a lock or ready queue.

Definition at line 1591 of file PerProcessorScheduler.cc.

Referenced by Rtc::deadlineInterrupt(), publishDeferredTimeAccounting(), RequestQueue::publishRequest(), ringIrqWorkDoorbell(), and Rtc::uninitialise().

Here is the caller graph for this function:

◆ ringIrqWorkDoorbell() [2/2]

void PerProcessorScheduler::ringIrqWorkDoorbell	(	SchedulerWorkerWake &	worker,
		bool	promptOwner = `true`
	)

Publishes one worker wake edge without touching a lock or ready queue.

Definition at line 1622 of file PerProcessorScheduler.cc.

References ringIrqWorkDoorbell().

◆ runTimeAccountingWorker()

int PerProcessorScheduler::runTimeAccountingWorker ( )

private

Definition at line 170 of file PerProcessorScheduler.cc.

◆ sampleUserReturnDiagnostics()

bool PerProcessorScheduler::sampleUserReturnDiagnostics ( )

inline

Selects one low-overhead, per-CPU user-return diagnostic sample.

Definition at line 183 of file PerProcessorScheduler.h.

Referenced by X64InterruptManager::returnFromInterrupt().

Here is the caller graph for this function:

◆ schedule()

void PerProcessorScheduler::schedule	(	Thread::Status	nextStatus = `Thread::Ready`,
		bool	dispatchEvents = `true`
	)

private

Picks another runnable thread and switches to it.

Definition at line 543 of file PerProcessorScheduler.cc.

References ProcessorBase::getInterrupts(), ProcessorBase::guardDeviceHardIrqOperation(), ProcessorBase::information(), and ProcessorBase::setInterrupts().

Referenced by blockCurrent(), idleUntilInterrupt(), serviceIrqWorkDoorbell(), and servicePendingScheduling().

Here is the caller graph for this function:

◆ scheduleWithInterruptState()

void PerProcessorScheduler::scheduleWithInterruptState	(	Thread::Status	nextStatus,
		bool	dispatchEvents,
		bool	previousInterruptState
	)

private

Definition at line 557 of file PerProcessorScheduler.cc.

◆ selectNext()

Thread * PerProcessorScheduler::selectNext ( Thread * current )

private

Definition at line 1476 of file PerProcessorScheduler.cc.

◆ serviceDeferredSubsystemException()

void PerProcessorScheduler::serviceDeferredSubsystemException	(	InterruptState &	state,
		bool	diagnosticSample = `false`
	)

private

Runs a raw-frame exception through its subsystem in ordinary context.

Definition at line 1941 of file PerProcessorScheduler.cc.

References ProcessorBase::getInterrupts(), Thread::getParent(), ProcessorBase::information(), Subsystem::resolveUserPageFault(), Thread::takeDeferredSubsystemException(), and Subsystem::threadException().

◆ serviceIrqWorkDoorbell()

void PerProcessorScheduler::serviceIrqWorkDoorbell ( )

Reschedules once from ordinary thread context during lifecycle work.

Definition at line 1659 of file PerProcessorScheduler.cc.

References ProcessorBase::information(), m_pSchedulingAlgorithm, and schedule().

◆ servicePendingScheduling()

void PerProcessorScheduler::servicePendingScheduling ( )

Services wakeups and reschedule requests at a safe scheduling boundary.

Definition at line 1674 of file PerProcessorScheduler.cc.

References ProcessorBase::information(), m_pSchedulingAlgorithm, and schedule().

Referenced by idleUntilInterrupt().

Here is the caller graph for this function:

◆ serviceProcessStopAtUserReturn()

bool PerProcessorScheduler::serviceProcessStopAtUserReturn ( ProcessStopGateMode mode = ProcessStopGateMode::StopOnly )

Waits until the current Process permits another userspace transition. Direct transitions also drain kernel Events which cannot use the ordinary user-stack-aware return service. Returns true when terminal Thread work wins while waiting.

Definition at line 1692 of file PerProcessorScheduler.cc.

References Thread::Continue, Thread::currentTimeAccountingMode(), ProcessorBase::executionContext(), ProcessorBase::getInterrupts(), Thread::getParent(), Thread::getUnwindState(), ProcessorBase::inDeviceHardIrq(), ProcessorBase::information(), Thread::m_EventWaiters, Thread::m_Lock, and Process::Reaped.

Referenced by Process::suspendInternal(), and HostedSyscallManager::syscall().

Here is the caller graph for this function:

◆ serviceUserReturnWork() [1/2]

bool PerProcessorScheduler::serviceUserReturnWork	(	InterruptState &	state,
		UserReturnFrame::Origin	origin = `UserReturnFrame::Origin::Interrupt`,
		bool	diagnosticSample = `false`
	)

Delivers pending Events and terminal work immediately before a user return, after the raw interrupt frame has released its C++ scopes.

Definition at line 1771 of file PerProcessorScheduler.cc.

References Thread::Continue, Thread::getUnwindState(), ProcessorBase::information(), and Thread::isTerminationDeferred().

Referenced by X64InterruptManager::returnFromInterrupt(), HostedSyscallManager::syscall(), and X64SyscallManager::syscall().

Here is the caller graph for this function:

◆ serviceUserReturnWork() [2/2]

bool PerProcessorScheduler::serviceUserReturnWork	(	SyscallState &	state,
		UserReturnFrame::Origin	origin = `UserReturnFrame::Origin::Syscall`,
		bool	diagnosticSample = `false`
	)

Delivers pending Events immediately before returning from a syscall.

Definition at line 1874 of file PerProcessorScheduler.cc.

References ProcessorBase::information().

◆ serviceWorkerWakeups()

void PerProcessorScheduler::serviceWorkerWakeups ( )

private

Definition at line 1631 of file PerProcessorScheduler.cc.

◆ setClockDeadline()

void PerProcessorScheduler::setClockDeadline ( uint64_t deadline )

Publish the shared timer's next deadline on CPU 0.

Definition at line 479 of file PerProcessorScheduler.cc.

References ProcessorBase::getInterrupts(), ProcessorBase::information(), ProcessorBase::informationAt(), and ProcessorBase::setInterrupts().

Referenced by Rtc::uninitialise().

Here is the caller graph for this function:

◆ setIdle()

void PerProcessorScheduler::setIdle ( Thread * pThread )

Definition at line 2250 of file PerProcessorScheduler.cc.

◆ stackDiscardCount() [1/2]

size_t PerProcessorScheduler::stackDiscardCount ( )

static

Monotonic count of explicit physical-stack discards.

Definition at line 1302 of file PerProcessorScheduler.cc.

◆ stackDiscardCount() [2/2]

size_t PerProcessorScheduler::stackDiscardCount ( StackDiscardReason reason )

static

Monotonic count for one explicit physical-stack discard reason.

Definition at line 1306 of file PerProcessorScheduler.cc.

◆ startNewThreadWorker()

void PerProcessorScheduler::startNewThreadWorker ( Process * pParent )

private

Definition at line 271 of file PerProcessorScheduler.cc.

◆ startTimeAccountingWorker()

void PerProcessorScheduler::startTimeAccountingWorker ( Process * pParent )

private

Definition at line 116 of file PerProcessorScheduler.cc.

◆ stopNewThreadWorker()

void PerProcessorScheduler::stopNewThreadWorker ( )

private

Definition at line 287 of file PerProcessorScheduler.cc.

◆ stopTimeAccountingWorker()

void PerProcessorScheduler::stopTimeAccountingWorker ( )

private

Definition at line 139 of file PerProcessorScheduler.cc.

◆ threadStatusChanged()

void PerProcessorScheduler::threadStatusChanged ( Thread * pThread )

Definition at line 1544 of file PerProcessorScheduler.cc.

◆ timeAccountingWorkerEntry()

int PerProcessorScheduler::timeAccountingWorkerEntry ( void * instance )

staticprivate

Definition at line 166 of file PerProcessorScheduler.cc.

◆ timer()

void PerProcessorScheduler::timer	(	uint64_t	delta,
		InterruptState &	state
	)

virtual

SchedulerTimerHandler callback.

Implements SchedulerTimerHandler.

Definition at line 1498 of file PerProcessorScheduler.cc.

References Thread::accountTimerTick(), Timer::deadlineInterrupt(), Machine::getTimer(), ProcessorBase::information(), Scheduler::instance(), and Scheduler::requestLoadAverageSample().

◆ unregisterWorkerWake()

void PerProcessorScheduler::unregisterWorkerWake ( SchedulerWorkerWake & worker )

Removes a worker wake registration after its worker has joined.

Definition at line 1607 of file PerProcessorScheduler.cc.

Referenced by Rtc::uninitialise().

Here is the caller graph for this function:

◆ updateOneShotTimer()

void PerProcessorScheduler::updateOneShotTimer ( )

private

Definition at line 509 of file PerProcessorScheduler.cc.

Friends And Related Symbol Documentation

◆ ProcessorThreadAllocator

friend class ProcessorThreadAllocator

friend

Definition at line 213 of file PerProcessorScheduler.h.

◆ Scheduler

friend class Scheduler

friend

Definition at line 211 of file PerProcessorScheduler.h.

◆ Thread

friend class Thread

friend

Definition at line 212 of file PerProcessorScheduler.h.

◆ WaitQueue

friend class WaitQueue

friend

Definition at line 214 of file PerProcessorScheduler.h.

Member Data Documentation

◆ DeferredReapPublicationClosed

constexpr size_t PerProcessorScheduler::DeferredReapPublicationClosed

staticconstexprprivate

Initial value:

= static_cast<size_t>(1)

<< ((sizeof(size_t) * 8) - 1)

Definition at line 318 of file PerProcessorScheduler.h.

◆ DeferredReapPublicationCountMask

constexpr size_t PerProcessorScheduler::DeferredReapPublicationCountMask = ~DeferredReapPublicationClosed

staticconstexprprivate

Definition at line 320 of file PerProcessorScheduler.h.

◆ m_AffinityAdmissionOpen

bool PerProcessorScheduler::m_AffinityAdmissionOpen = false

private

Definition at line 306 of file PerProcessorScheduler.h.

◆ m_AffinityHead

Thread* PerProcessorScheduler::m_AffinityHead = nullptr

private

Definition at line 303 of file PerProcessorScheduler.h.

◆ m_AffinityQueueLock

Spinlock PerProcessorScheduler::m_AffinityQueueLock

private

Definition at line 302 of file PerProcessorScheduler.h.

◆ m_AffinityRequests

Atomic<size_t> PerProcessorScheduler::m_AffinityRequests

private

Definition at line 305 of file PerProcessorScheduler.h.

◆ m_AffinityTail

Thread* PerProcessorScheduler::m_AffinityTail = nullptr

private

Definition at line 304 of file PerProcessorScheduler.h.

◆ m_ClockDeadline

Atomic<uint64_t> PerProcessorScheduler::m_ClockDeadline

private

Definition at line 299 of file PerProcessorScheduler.h.

◆ m_DeferredThreadReapPublicationState

Atomic<size_t> PerProcessorScheduler::m_DeferredThreadReapPublicationState

private

Definition at line 291 of file PerProcessorScheduler.h.

◆ m_DeferredThreadReaps

IntrusiveMpscQueue<DeferredThreadReapNode, &DeferredThreadReapNode::next> PerProcessorScheduler::m_DeferredThreadReaps

private

Definition at line 289 of file PerProcessorScheduler.h.

◆ m_DeferredThreadReapStub

DeferredThreadReapNode PerProcessorScheduler::m_DeferredThreadReapStub

private

Definition at line 288 of file PerProcessorScheduler.h.

◆ m_DelayedNewThreadData

List<void*> PerProcessorScheduler::m_DelayedNewThreadData

private

Definition at line 283 of file PerProcessorScheduler.h.

◆ m_IdleWakeRequested

bool PerProcessorScheduler::m_IdleWakeRequested = false

private

Definition at line 325 of file PerProcessorScheduler.h.

◆ m_IrqWorkDoorbell

Atomic<size_t> PerProcessorScheduler::m_IrqWorkDoorbell

private

Definition at line 296 of file PerProcessorScheduler.h.

◆ m_IrqWorkLock

Spinlock PerProcessorScheduler::m_IrqWorkLock

private

Definition at line 300 of file PerProcessorScheduler.h.

◆ m_LogicalCpu

size_t PerProcessorScheduler::m_LogicalCpu = ~size_t(0)

private

Definition at line 307 of file PerProcessorScheduler.h.

◆ m_nDeferredThreadReaps

Atomic<size_t> PerProcessorScheduler::m_nDeferredThreadReaps

private

Definition at line 290 of file PerProcessorScheduler.h.

◆ m_NewThreadAdmissionOpen

bool PerProcessorScheduler::m_NewThreadAdmissionOpen

private

Definition at line 284 of file PerProcessorScheduler.h.

◆ m_NewThreadData

List<void*> PerProcessorScheduler::m_NewThreadData

private

Definition at line 282 of file PerProcessorScheduler.h.

◆ m_NewThreadDataCondition

ConditionVariable PerProcessorScheduler::m_NewThreadDataCondition

private

Definition at line 280 of file PerProcessorScheduler.h.

◆ m_NewThreadDataLock

Mutex PerProcessorScheduler::m_NewThreadDataLock

private

Definition at line 279 of file PerProcessorScheduler.h.

◆ m_NewThreadWorker

OwnedThread PerProcessorScheduler::m_NewThreadWorker

private

Definition at line 286 of file PerProcessorScheduler.h.

◆ m_NextLoadSampleDeadline

uint64_t PerProcessorScheduler::m_NextLoadSampleDeadline = 0

private

Definition at line 311 of file PerProcessorScheduler.h.

◆ m_NominalQuantumNs

uint64_t PerProcessorScheduler::m_NominalQuantumNs = 0

private

Definition at line 309 of file PerProcessorScheduler.h.

◆ m_OneShotTimer

bool PerProcessorScheduler::m_OneShotTimer = false

private

Definition at line 312 of file PerProcessorScheduler.h.

◆ m_PhysicalCpu

size_t PerProcessorScheduler::m_PhysicalCpu = 0

private

Definition at line 308 of file PerProcessorScheduler.h.

◆ m_pIdleThread

Thread* PerProcessorScheduler::m_pIdleThread

private

Definition at line 324 of file PerProcessorScheduler.h.

◆ m_pSchedulingAlgorithm

SchedulingAlgorithm* PerProcessorScheduler::m_pSchedulingAlgorithm

private

The current SchedulingAlgorithm

Definition at line 277 of file PerProcessorScheduler.h.

Referenced by addThread(), addThread(), idleUntilInterrupt(), initialise(), serviceIrqWorkDoorbell(), and servicePendingScheduling().

◆ m_pWorkerWakeHead

SchedulerWorkerWake* PerProcessorScheduler::m_pWorkerWakeHead = nullptr

private

Definition at line 301 of file PerProcessorScheduler.h.

◆ m_QuantumDeadline

uint64_t PerProcessorScheduler::m_QuantumDeadline = 0

private

Definition at line 310 of file PerProcessorScheduler.h.

◆ m_RemotePromptPending

Atomic<size_t> PerProcessorScheduler::m_RemotePromptPending

private

Definition at line 298 of file PerProcessorScheduler.h.

◆ m_ReschedulePending

Atomic<size_t> PerProcessorScheduler::m_ReschedulePending

private

Definition at line 297 of file PerProcessorScheduler.h.

◆ m_SchedulerTickCounter

size_t PerProcessorScheduler::m_SchedulerTickCounter = 0

private

Definition at line 326 of file PerProcessorScheduler.h.

◆ m_StopNewThreadWorker

bool PerProcessorScheduler::m_StopNewThreadWorker

private

Definition at line 285 of file PerProcessorScheduler.h.

◆ m_StopTimeAccountingWorker

Atomic<size_t> PerProcessorScheduler::m_StopTimeAccountingWorker

private

Definition at line 292 of file PerProcessorScheduler.h.

◆ m_TimeAccountingState

DeferredTimeAccountingWorkerState PerProcessorScheduler::m_TimeAccountingState

private

Definition at line 287 of file PerProcessorScheduler.h.

◆ m_TimeAccountingWorker

OwnedThread PerProcessorScheduler::m_TimeAccountingWorker

private

Definition at line 293 of file PerProcessorScheduler.h.

◆ m_TimeAccountingWorkerWaiters

WaitQueue PerProcessorScheduler::m_TimeAccountingWorkerWaiters

private

Definition at line 294 of file PerProcessorScheduler.h.

◆ m_TimeAccountingWorkerWake

SchedulerWorkerWake PerProcessorScheduler::m_TimeAccountingWorkerWake

private

Definition at line 295 of file PerProcessorScheduler.h.

◆ m_UserReturnDiagnosticSequence

size_t PerProcessorScheduler::m_UserReturnDiagnosticSequence = 0

private

Definition at line 327 of file PerProcessorScheduler.h.

The documentation for this class was generated from the following files:

Public Types

Public Member Functions

Static Public Member Functions

Private Member Functions

Static Private Member Functions

Private Attributes

Static Private Attributes

Friends

Detailed Description

Member Enumeration Documentation

◆ ProcessStopGateMode

◆ StackDiscardReason

Constructor & Destructor Documentation

◆ PerProcessorScheduler() [1/2]

◆ ~PerProcessorScheduler()

◆ PerProcessorScheduler() [2/2]

Member Function Documentation

◆ abandonCurrentThreadStack()

◆ addThread() [1/2]

◆ addThread() [2/2]

◆ armLocalQuantumIfNeeded()

◆ blockCurrent()

◆ checkEventState() [1/3]

◆ checkEventState() [2/3]

◆ checkEventState() [3/3]

◆ commitCurrentThreadExit()

◆ commitUserReturnTerminalState()

◆ deleteThread()

◆ deleteThreadThenRestoreState()

◆ drainAffinityRequests()

◆ drainDeferredThreadReaps()

◆ enqueueAffinity()

◆ eventHandlerReturned()

◆ finishCurrentThreadExit()

◆ idleUntilInterrupt()

◆ initialise()

◆ isIdleThread()

◆ killCurrentThread()

◆ logicalCpu()

◆ nominalQuantumNs()

◆ operator=()

◆ processorAddThread()

◆ programOneShotTimer()

◆ prompt()

◆ publishDeferredThreadReap()

◆ publishDeferredTimeAccounting()

◆ publishReadyFromWait()

◆ registerWorkerWake()

◆ removeThread()

◆ requestCurrentThreadExitToIdle()

◆ requestIdleThreadWakeup()

◆ ringIrqWorkDoorbell() [1/2]

◆ ringIrqWorkDoorbell() [2/2]

◆ runTimeAccountingWorker()

◆ sampleUserReturnDiagnostics()

◆ schedule()

◆ scheduleWithInterruptState()

◆ selectNext()

◆ serviceDeferredSubsystemException()

◆ serviceIrqWorkDoorbell()

◆ servicePendingScheduling()

◆ serviceProcessStopAtUserReturn()

◆ serviceUserReturnWork() [1/2]

◆ serviceUserReturnWork() [2/2]

◆ serviceWorkerWakeups()

◆ setClockDeadline()

◆ setIdle()

◆ stackDiscardCount() [1/2]

◆ stackDiscardCount() [2/2]

◆ startNewThreadWorker()

◆ startTimeAccountingWorker()

◆ stopNewThreadWorker()

◆ stopTimeAccountingWorker()

◆ threadStatusChanged()

◆ timeAccountingWorkerEntry()

◆ timer()

◆ unregisterWorkerWake()

◆ updateOneShotTimer()

Friends And Related Symbol Documentation

◆ ProcessorThreadAllocator