#include <IrqHandlerRegistry.h>

Collaboration diagram for IrqHandlerRegistry:

Classes
struct	ActiveDispatch

struct	AdmissionCutoff

struct	AdmissionCutoffCleanup

struct	DispatchCleanup

struct	HandlerSlot

struct	LineConfiguration

struct	MixedAdmissionCutoffs

struct	ThreadedActionMutationCleanup

struct	ThreadedDispatchResult

Public Types
enum class	UnregisterResult { Completed , Deferred , Rejected , NotFound }

enum class	LineMode { Empty , Threaded , HardOnly , Mixed }

Public Member Functions
bool	registerThreadedHandler (uint8_t irq, IrqHandler *handler)

bool	registerThreadedHandler (uint8_t irq, IrqHandler *handler, const IrqPolicy &policy)

bool	registerHardHandler (uint8_t irq, HardIrqHandler *handler)

bool	registerHardHandler (uint8_t irq, HardIrqHandler *handler, const IrqPolicy &policy)

UnregisterResult	unregisterHandler (uint8_t irq, IrqHandlerBase *handler)

UnregisterResult	unregisterHandler (uint8_t irq, IrqHandlerBase *handler, LineMode &removedDelivery)

bool	dispatchHard (uint8_t irq, InterruptState &state, HardIrqDisposition &disposition, HardIrqHandler *onlyHandler=nullptr, size_t dispatchGeneration=0, irq_id_t callbackId=0)

bool	dispatchHard (uint8_t irq, InterruptState &state, HardIrqDisposition &disposition, HardIrqHandler *onlyHandler, size_t dispatchGeneration, AdmissionCutoff admissionCutoff, irq_id_t callbackId=0)

bool	hardLineQuarantined (uint8_t irq) const

bool	publishThreadedDispatch (uint8_t irq, size_t dispatchGeneration)

bool	publishThreadedDispatch (uint8_t irq, size_t dispatchGeneration, AdmissionCutoff admissionCutoff)

bool	captureAdmissionCutoff (uint8_t irq, AdmissionCutoff &cutoff)

bool	captureMixedAdmissionCutoffs (uint8_t irq, MixedAdmissionCutoffs &cutoffs)

void	releaseAdmissionCutoff (AdmissionCutoff admissionCutoff)

void	invalidateThreadedLine (uint8_t irq, size_t throughGeneration)

void	invalidateThreadedGenerationFromInterrupt (uint8_t irq, size_t throughGeneration)

bool	dispatchThreaded (uint8_t irq, size_t dispatchGeneration, ThreadedDispatchResult &result, IrqHandler *onlyHandler=nullptr, irq_id_t callbackId=0)

size_t	handlerCount (uint8_t irq)

bool	containsHandler (uint8_t irq, IrqHandlerBase *handler) const

LineMode	lineMode (uint8_t irq)

bool	snapshotLineConfiguration (uint8_t irq, LineConfiguration &configuration) const

size_t	hardDispatchState (uint8_t irq, size_t &exactGeneration) const

size_t	threadedDispatchState (uint8_t irq, uintptr_t &exactHandlerIdentity) const

Private Types
enum class	Delivery : size_t { Threaded , HardOnly }

enum class	SlotMode : size_t { Empty = 0 , Enabled , Draining , Cancelling , Closed , Retiring , Tombstone }

enum class	QuiescedLane : size_t { Controller , Callback , Retirement , Count }

Private Member Functions
bool	registerHandler (uint8_t irq, IrqHandlerBase *handler, Delivery delivery, size_t policy)

bool	acquireOccurrenceReaderLeases (uint8_t irq, size_t readerBank, size_t count)

void	releaseOccurrenceReaderLeases (uint8_t irq, size_t readerBank, size_t count)

void	beginAdmissionCutoffCleanup (AdmissionCutoffCleanup &cleanup)

void	finishAdmissionCutoffCleanup (AdmissionCutoffCleanup &cleanup)

void	beginMutation ()

void	finishMutation ()

void	beginThreadedActionMutation (ThreadedActionMutationCleanup &cleanup)

void	finishThreadedActionMutation (ThreadedActionMutationCleanup &cleanup)

void	completeThreadedActionMutation ()

bool	retireSlot (HandlerSlot &slot, size_t expectedPublication, IrqHandlerBase *expectedHandler)

bool	retireSlotOrObserveClosed (HandlerSlot &slot, size_t expectedPublication, IrqHandlerBase *expectedHandler)

bool	closeSlotAdmission (HandlerSlot &slot, size_t expectedPublication, size_t &closedPublication)

void	tryReclaimTombstones (uint8_t irq)

bool	occurrencePrecedesRetirement (const HandlerSlot &slot, AdmissionCutoff admissionCutoff) const

bool	publishDispatch (DispatchCleanup &cleanup, size_t controllerGeneration)

bool	unpublishDispatch (DispatchCleanup &cleanup, bool required)

bool	acquireFinalizationGate (HandlerSlot &slot, bool canWait)

bool	pinActionMutation (HandlerSlot &slot, size_t publication, DispatchCleanup &cleanup, Thread *thread)

void	unpinActionMutation (DispatchCleanup &cleanup, Thread *thread)

bool	hasActiveDispatch (HandlerSlot &slot, size_t admittedPublication) const

bool	findCurrentDispatch (void owner, HandlerSlot target, size_t targetPublication, bool &callbackContext) const

bool	threadedGenerationValid (uint8_t irq, size_t generation) const

void	publishSlotQuiesced (HandlerSlot &slot, uint8_t irq, size_t dispatchGeneration, QuiescedLane lane)

void	publishSlotQuiescedValue (HandlerSlot &slot, uint8_t irq, size_t dispatchGeneration, QuiescedLane lane)

Static Private Member Functions
static size_t	makePublication (size_t generation, uint8_t irq, SlotMode mode, Delivery delivery)

static size_t	generationOf (size_t publication)

static uint8_t	irqOf (size_t publication)

static SlotMode	modeOf (size_t publication)

static Delivery	deliveryOf (size_t publication)

static bool	generationReached (size_t current, size_t target)

static size_t	encodePolicy (const IrqPolicy *policy)

static void	decodePolicy (size_t policy, LineConfiguration &configuration)

static bool	mixedPoliciesCompatible (size_t first, size_t second)

static size_t	effectiveMixedPolicy (size_t hard, size_t threaded)

static LineMode	lineModeForDelivery (Delivery delivery)

static void	abandonAdmissionCutoff (void *context)

static void	abandonThreadedActionMutation (void *context)

static void	abandonDispatch (void *context)

static void	restoreDispatchInterruptState (DispatchCleanup &dispatch)

static bool	canWaitForActionFinalization ()

static void	releaseFinalizationGate (HandlerSlot &slot)

static void *	currentDispatchOwner ()

static size_t *	quiescedLane (HandlerSlot &slot, QuiescedLane lane)

static const size_t *	quiescedLane (const HandlerSlot &slot, QuiescedLane lane)

static bool	hasQuiescedGeneration (const HandlerSlot &slot)

Private Attributes
HandlerSlot	m_Handlers [MaxHandlerSlots]

ActiveDispatch	m_ActiveDispatches [MaxActiveDispatches]

size_t	m_HardHandoffEpochs [GraceBucketCount]

size_t	m_ThreadedInvalidationGenerations [IrqCount]

size_t	m_ThreadedActionMutationGeneration

size_t	m_ThreadedActionMutationWriters

size_t	m_OccurrenceEpochs [GraceBucketCount]

size_t	m_OccurrenceReaders [GraceBucketCount][2]

size_t	m_OccurrenceBoundaryLocks [GraceBucketCount]

Spinlock	m_HandlerLock

size_t	m_AdmissionEpoch

size_t	m_MutationGeneration

size_t	m_MutationWriters

Static Private Attributes
static constexpr size_t	MaxHandlerSlots = 64

static constexpr size_t	MaxActiveDispatches = 64

static constexpr size_t	IrqCount = 256

static constexpr size_t	GraceBucketCount = 16

static constexpr uint8_t	InvalidIrq = 0xFF

static constexpr size_t	QuiescedLaneCount = static_cast<size_t>(QuiescedLane::Count)

static constexpr size_t	ModeBits = 3

static constexpr size_t	ModeMask = (1 << ModeBits) - 1

static constexpr size_t	DeliveryBits = 1

static constexpr size_t	DeliveryShift = ModeBits

static constexpr size_t	DeliveryMask = ((1 << DeliveryBits) - 1) << DeliveryShift

static constexpr size_t	IrqShift = DeliveryShift + DeliveryBits

static constexpr size_t	IrqMask = 0xFF << IrqShift

static constexpr size_t	GenerationShift = IrqShift + 8

static constexpr size_t	MaximumPublicationGeneration = ~static_cast<size_t>(0) >> GenerationShift

static constexpr size_t	PolicyValid = 1U << 0

static constexpr size_t	PolicyTriggerShift = 1

static constexpr size_t	PolicyControllerAckShift = 3

static constexpr size_t	PolicyLineReleaseShift = 5

static constexpr size_t	PolicyTriggerMask = 3U << PolicyTriggerShift

static constexpr size_t	PolicyControllerAckMask = 3U << PolicyControllerAckShift

static constexpr size_t	PolicyLineReleaseMask = 1U << PolicyLineReleaseShift

static constexpr size_t	PolicyMixedCompatibilityMask

static constexpr size_t	LineSnapshotAttempts = 2

Detailed Description

Allocation-free IRQ callback registry with synchronous removal.

Dispatch never takes the registry's writer lock. It pins an atomically published slot and revalidates that publication before entering the callback. Removal first closes admission and then drains callbacks which committed their pins before that transition. Callback release only clears atomic hazard state; a synchronous unregistering thread owns any cooperative scheduling needed while it rechecks the drain predicate.

Definition at line 35 of file IrqHandlerRegistry.h.

Member Enumeration Documentation

◆ Delivery

enum class IrqHandlerRegistry::Delivery : size_t

strongprivate

Definition at line 285 of file IrqHandlerRegistry.h.

◆ LineMode

enum class IrqHandlerRegistry::LineMode

strong

Definition at line 60 of file IrqHandlerRegistry.h.

◆ QuiescedLane

enum class IrqHandlerRegistry::QuiescedLane : size_t

strongprivate

Definition at line 307 of file IrqHandlerRegistry.h.

◆ SlotMode

enum class IrqHandlerRegistry::SlotMode : size_t

strongprivate

Definition at line 297 of file IrqHandlerRegistry.h.

◆ UnregisterResult

enum class IrqHandlerRegistry::UnregisterResult

strong

Definition at line 53 of file IrqHandlerRegistry.h.

Constructor & Destructor Documentation

◆ IrqHandlerRegistry()

IrqHandlerRegistry::IrqHandlerRegistry ( )

Definition at line 28 of file IrqHandlerRegistry.cc.

Member Function Documentation

◆ abandonAdmissionCutoff()

void IrqHandlerRegistry::abandonAdmissionCutoff ( void * context )

staticprivate

Definition at line 1242 of file IrqHandlerRegistry.cc.

◆ abandonDispatch()

void IrqHandlerRegistry::abandonDispatch ( void * context )

staticprivate

Definition at line 737 of file IrqHandlerRegistry.cc.

◆ abandonThreadedActionMutation()

void IrqHandlerRegistry::abandonThreadedActionMutation ( void * context )

staticprivate

Definition at line 325 of file IrqHandlerRegistry.cc.

◆ acquireFinalizationGate()

bool IrqHandlerRegistry::acquireFinalizationGate	(	HandlerSlot &	slot,
		bool	canWait
	)

private

Definition at line 345 of file IrqHandlerRegistry.cc.

◆ acquireOccurrenceReaderLeases()

bool IrqHandlerRegistry::acquireOccurrenceReaderLeases	(	uint8_t	irq,
		size_t	readerBank,
		size_t	count
	)

private

Definition at line 1157 of file IrqHandlerRegistry.cc.

◆ beginAdmissionCutoffCleanup()

void IrqHandlerRegistry::beginAdmissionCutoffCleanup ( AdmissionCutoffCleanup & cleanup )

private

Definition at line 1212 of file IrqHandlerRegistry.cc.

◆ beginMutation()

void IrqHandlerRegistry::beginMutation ( )

private

Definition at line 280 of file IrqHandlerRegistry.cc.

◆ beginThreadedActionMutation()

void IrqHandlerRegistry::beginThreadedActionMutation ( ThreadedActionMutationCleanup & cleanup )

private

Definition at line 293 of file IrqHandlerRegistry.cc.

◆ canWaitForActionFinalization()

bool IrqHandlerRegistry::canWaitForActionFinalization ( )

staticprivate

Definition at line 336 of file IrqHandlerRegistry.cc.

◆ captureAdmissionCutoff()

bool IrqHandlerRegistry::captureAdmissionCutoff	(	uint8_t	irq,
		AdmissionCutoff &	cutoff
	)

Captures one occurrence cutoff before controller delivery is sampled. A false return leaves cutoff empty and transfers no lease.

Definition at line 1086 of file IrqHandlerRegistry.cc.

Referenced by dispatchHard(), dispatchThreaded(), Pic::interrupt(), and publishThreadedDispatch().

Here is the caller graph for this function:

◆ captureMixedAdmissionCutoffs()

bool IrqHandlerRegistry::captureMixedAdmissionCutoffs	(	uint8_t	irq,
		MixedAdmissionCutoffs &	cutoffs
	)

Captures independent hard and threaded leases for one occurrence.

Each cutoff must be consumed by its dispatch operation or released separately. Both leases have the same occurrence and admission epochs. A false return leaves both cutoffs empty and transfers no leases.

Definition at line 1123 of file IrqHandlerRegistry.cc.

Referenced by Pic::interrupt().

Here is the caller graph for this function:

◆ closeSlotAdmission()

bool IrqHandlerRegistry::closeSlotAdmission	(	HandlerSlot &	slot,
		size_t	expectedPublication,
		size_t &	closedPublication
	)

private

Definition at line 400 of file IrqHandlerRegistry.cc.

◆ completeThreadedActionMutation()

void IrqHandlerRegistry::completeThreadedActionMutation ( )

private

Definition at line 315 of file IrqHandlerRegistry.cc.

◆ containsHandler()

bool IrqHandlerRegistry::containsHandler	(	uint8_t	irq,
		IrqHandlerBase *	handler
	)		const

Returns whether this handler is currently registered on the line.

Definition at line 1954 of file IrqHandlerRegistry.cc.

◆ currentDispatchOwner()

void * IrqHandlerRegistry::currentDispatchOwner ( )

staticprivate

Definition at line 820 of file IrqHandlerRegistry.cc.

◆ decodePolicy()

void IrqHandlerRegistry::decodePolicy	(	size_t	policy,
		LineConfiguration &	configuration
	)

staticprivate

Definition at line 248 of file IrqHandlerRegistry.cc.

◆ deliveryOf()

IrqHandlerRegistry::Delivery IrqHandlerRegistry::deliveryOf ( size_t publication )

staticprivate

Definition at line 71 of file IrqHandlerRegistry.cc.

◆ dispatchHard() [1/2]

bool IrqHandlerRegistry::dispatchHard	(	uint8_t	irq,
		InterruptState &	state,
		HardIrqDisposition &	disposition,
		HardIrqHandler *	onlyHandler,
		size_t	dispatchGeneration,
		AdmissionCutoff	admissionCutoff,
		irq_id_t	callbackId = `0`
	)

Definition at line 1253 of file IrqHandlerRegistry.cc.

◆ dispatchHard() [2/2]

bool IrqHandlerRegistry::dispatchHard	(	uint8_t	irq,
		InterruptState &	state,
		HardIrqDisposition &	disposition,
		HardIrqHandler *	onlyHandler = `nullptr`,
		size_t	dispatchGeneration = `0`,
		irq_id_t	callbackId = `0`
	)

Dispatches all enabled handlers for an IRQ.

Returns true if at least one callback was admitted. disposition aggregates callback results, with KeepMasked dominating Handled and Handled dominating NotHandled.

Definition at line 1074 of file IrqHandlerRegistry.cc.

References captureAdmissionCutoff(), and dispatchHard().

Referenced by dispatchHard(), and Pic::interrupt().

Here is the caller graph for this function:

◆ dispatchThreaded()

bool IrqHandlerRegistry::dispatchThreaded	(	uint8_t	irq,
		size_t	dispatchGeneration,
		ThreadedDispatchResult &	result,
		IrqHandler *	onlyHandler = `nullptr`,
		irq_id_t	callbackId = `0`
	)

Dispatches handlers admitted by publishThreadedDispatch() without exposing interrupted state.

Returns true when at least one callback was admitted and writes the detached callback outcome to result.

A Quiesced callback permits line rearm but remains distinct from a callback which actually handled the occurrence.

Definition at line 1640 of file IrqHandlerRegistry.cc.

References captureAdmissionCutoff(), ProcessorBase::getInterrupts(), ProcessorBase::information(), Scheduler::instance(), Quiesced, ProcessorBase::setInterrupts(), and Scheduler::yield().

◆ effectiveMixedPolicy()

size_t IrqHandlerRegistry::effectiveMixedPolicy	(	size_t	hard,
		size_t	threaded
	)

staticprivate

Definition at line 268 of file IrqHandlerRegistry.cc.

◆ encodePolicy()

size_t IrqHandlerRegistry::encodePolicy ( const IrqPolicy * policy )

staticprivate

Definition at line 238 of file IrqHandlerRegistry.cc.

◆ findCurrentDispatch()

bool IrqHandlerRegistry::findCurrentDispatch	(	void *	owner,
		HandlerSlot *	target,
		size_t	targetPublication,
		bool &	callbackContext
	)		const

private

Definition at line 789 of file IrqHandlerRegistry.cc.

◆ finishAdmissionCutoffCleanup()

void IrqHandlerRegistry::finishAdmissionCutoffCleanup ( AdmissionCutoffCleanup & cleanup )

private

Definition at line 1226 of file IrqHandlerRegistry.cc.

◆ finishMutation()

void IrqHandlerRegistry::finishMutation ( )

private

Definition at line 286 of file IrqHandlerRegistry.cc.

◆ finishThreadedActionMutation()

void IrqHandlerRegistry::finishThreadedActionMutation ( ThreadedActionMutationCleanup & cleanup )

private

Definition at line 304 of file IrqHandlerRegistry.cc.

◆ generationOf()

size_t IrqHandlerRegistry::generationOf ( size_t publication )

staticprivate

Definition at line 59 of file IrqHandlerRegistry.cc.

◆ generationReached()

bool IrqHandlerRegistry::generationReached	(	size_t	current,
		size_t	target
	)

staticprivate

Definition at line 75 of file IrqHandlerRegistry.cc.

◆ handlerCount()

size_t IrqHandlerRegistry::handlerCount ( uint8_t irq )

Definition at line 1942 of file IrqHandlerRegistry.cc.

◆ hardDispatchState()

size_t IrqHandlerRegistry::hardDispatchState	(	uint8_t	irq,
		size_t &	exactGeneration
	)		const

Lock-free debugger query for committed hard callbacks on one line.

Returns the active count and writes the exact controller generation only when one callback is active. Multiple callbacks make the generation ambiguous and write zero.

Definition at line 2073 of file IrqHandlerRegistry.cc.

Referenced by Pic::snapshotIrqLines().

Here is the caller graph for this function:

◆ hardLineQuarantined()

bool IrqHandlerRegistry::hardLineQuarantined ( uint8_t irq ) const

Reports a sticky KeepMasked result from any live hard source.

Definition at line 1418 of file IrqHandlerRegistry.cc.

References IrqHandlerRegistry::HandlerSlot::hardHandoffState, and m_HardHandoffEpochs.

◆ hasActiveDispatch()

bool IrqHandlerRegistry::hasActiveDispatch	(	HandlerSlot &	slot,
		size_t	admittedPublication
	)		const

private

Definition at line 768 of file IrqHandlerRegistry.cc.

◆ hasQuiescedGeneration()

bool IrqHandlerRegistry::hasQuiescedGeneration ( const HandlerSlot & slot )

staticprivate

Definition at line 101 of file IrqHandlerRegistry.cc.

◆ invalidateThreadedGenerationFromInterrupt()

void IrqHandlerRegistry::invalidateThreadedGenerationFromInterrupt	(	uint8_t	irq,
		size_t	throughGeneration
	)

Invalidates a generation in bounded hard-IRQ context.

The controller serialises this with publication for the physical line. Token and tombstone cleanup is deferred to workers or full line invalidation.

Definition at line 164 of file IrqHandlerRegistry.cc.

◆ invalidateThreadedLine()

void IrqHandlerRegistry::invalidateThreadedLine	(	uint8_t	irq,
		size_t	throughGeneration
	)

Invalidates queued threaded work from an ended controller lifetime.

Definition at line 179 of file IrqHandlerRegistry.cc.

References ProcessorBase::information().

Referenced by VirtIrqManager::registerPciMsixIrqHandlers(), Pic::shutdownThreaded(), and VirtIrqManager::unregisterHandler().

Here is the caller graph for this function:

◆ irqOf()

uint8_t IrqHandlerRegistry::irqOf ( size_t publication )

staticprivate

Definition at line 63 of file IrqHandlerRegistry.cc.

◆ lineMode()

IrqHandlerRegistry::LineMode IrqHandlerRegistry::lineMode ( uint8_t irq )

Returns the delivery type of an enabled physical line.

Definition at line 1970 of file IrqHandlerRegistry.cc.

◆ lineModeForDelivery()

IrqHandlerRegistry::LineMode IrqHandlerRegistry::lineModeForDelivery ( Delivery delivery )

staticprivate

Definition at line 276 of file IrqHandlerRegistry.cc.

◆ makePublication()

size_t IrqHandlerRegistry::makePublication	(	size_t	generation,
		uint8_t	irq,
		SlotMode	mode,
		Delivery	delivery
	)

staticprivate

Definition at line 53 of file IrqHandlerRegistry.cc.

◆ mixedPoliciesCompatible()

bool IrqHandlerRegistry::mixedPoliciesCompatible	(	size_t	first,
		size_t	second
	)

staticprivate

Definition at line 260 of file IrqHandlerRegistry.cc.

◆ modeOf()

IrqHandlerRegistry::SlotMode IrqHandlerRegistry::modeOf ( size_t publication )

staticprivate

Definition at line 67 of file IrqHandlerRegistry.cc.

◆ occurrencePrecedesRetirement()

bool IrqHandlerRegistry::occurrencePrecedesRetirement	(	const HandlerSlot &	slot,
		AdmissionCutoff	admissionCutoff
	)		const

private

Definition at line 492 of file IrqHandlerRegistry.cc.

◆ pinActionMutation()

bool IrqHandlerRegistry::pinActionMutation	(	HandlerSlot &	slot,
		size_t	publication,
		DispatchCleanup &	cleanup,
		Thread *	thread
	)

private

Definition at line 374 of file IrqHandlerRegistry.cc.

◆ publishDispatch()

bool IrqHandlerRegistry::publishDispatch	(	DispatchCleanup &	cleanup,
		size_t	controllerGeneration
	)

private

Definition at line 612 of file IrqHandlerRegistry.cc.

◆ publishSlotQuiesced()

void IrqHandlerRegistry::publishSlotQuiesced	(	HandlerSlot &	slot,
		uint8_t	irq,
		size_t	dispatchGeneration,
		QuiescedLane	lane
	)

private

Definition at line 110 of file IrqHandlerRegistry.cc.

◆ publishSlotQuiescedValue()

void IrqHandlerRegistry::publishSlotQuiescedValue	(	HandlerSlot &	slot,
		uint8_t	irq,
		size_t	dispatchGeneration,
		QuiescedLane	lane
	)

private

Definition at line 127 of file IrqHandlerRegistry.cc.

◆ publishThreadedDispatch() [1/2]

bool IrqHandlerRegistry::publishThreadedDispatch	(	uint8_t	irq,
		size_t	dispatchGeneration
	)

Records the exact threaded-handler publications admitted for one physical occurrence.

The nonzero generation is supplied again by the line worker. Repeated occurrences may coalesce to the newest generation, but a handler which was registered after this call is never admitted retroactively. A controller serializes publication and invalidation for each physical line; worker consumption remains concurrent.

Definition at line 1456 of file IrqHandlerRegistry.cc.

References captureAdmissionCutoff(), and publishThreadedDispatch().

Referenced by Pic::interrupt(), and publishThreadedDispatch().

Here is the caller graph for this function:

◆ publishThreadedDispatch() [2/2]

bool IrqHandlerRegistry::publishThreadedDispatch	(	uint8_t	irq,
		size_t	dispatchGeneration,
		AdmissionCutoff	admissionCutoff
	)

Definition at line 1464 of file IrqHandlerRegistry.cc.

◆ quiescedLane() [1/2]

const size_t * IrqHandlerRegistry::quiescedLane	(	const HandlerSlot &	slot,
		QuiescedLane	lane
	)

staticprivate

Definition at line 95 of file IrqHandlerRegistry.cc.

◆ quiescedLane() [2/2]

size_t * IrqHandlerRegistry::quiescedLane	(	HandlerSlot &	slot,
		QuiescedLane	lane
	)

staticprivate

Definition at line 89 of file IrqHandlerRegistry.cc.

◆ registerHandler()

bool IrqHandlerRegistry::registerHandler	(	uint8_t	irq,
		IrqHandlerBase *	handler,
		Delivery	delivery,
		size_t	policy
	)

private

Definition at line 826 of file IrqHandlerRegistry.cc.

◆ registerHardHandler() [1/2]

bool IrqHandlerRegistry::registerHardHandler	(	uint8_t	irq,
		HardIrqHandler *	handler
	)

Publishes an explicit hard-IRQ handler.

Definition at line 906 of file IrqHandlerRegistry.cc.

Referenced by Pic::registerHardIsaIrqHandler(), and Pic::registerHardPciIrqHandler().

Here is the caller graph for this function:

◆ registerHardHandler() [2/2]

bool IrqHandlerRegistry::registerHardHandler	(	uint8_t	irq,
		HardIrqHandler *	handler,
		const IrqPolicy &	policy
	)

Publishes a typed hard-IRQ handler and line policy.

Definition at line 910 of file IrqHandlerRegistry.cc.

◆ registerThreadedHandler() [1/2]

bool IrqHandlerRegistry::registerThreadedHandler	(	uint8_t	irq,
		IrqHandler *	handler
	)

Publishes an ordinary thread-context handler.

Definition at line 896 of file IrqHandlerRegistry.cc.

Referenced by Pic::registerIsaIrqHandler(), Pic::registerPciIrqHandler(), and VirtIrqManager::registerPciMsixIrqHandlers().

Here is the caller graph for this function:

◆ registerThreadedHandler() [2/2]

bool IrqHandlerRegistry::registerThreadedHandler	(	uint8_t	irq,
		IrqHandler *	handler,
		const IrqPolicy &	policy
	)

Publishes a typed thread-context handler and line policy.

Definition at line 900 of file IrqHandlerRegistry.cc.

◆ releaseAdmissionCutoff()

void IrqHandlerRegistry::releaseAdmissionCutoff ( AdmissionCutoff admissionCutoff )

Releases a cutoff which was not consumed by a dispatch operation.

Definition at line 1197 of file IrqHandlerRegistry.cc.

◆ releaseFinalizationGate()

void IrqHandlerRegistry::releaseFinalizationGate ( HandlerSlot & slot )

staticprivate

Definition at line 370 of file IrqHandlerRegistry.cc.

◆ releaseOccurrenceReaderLeases()

void IrqHandlerRegistry::releaseOccurrenceReaderLeases	(	uint8_t	irq,
		size_t	readerBank,
		size_t	count
	)

private

Definition at line 1175 of file IrqHandlerRegistry.cc.

◆ restoreDispatchInterruptState()

void IrqHandlerRegistry::restoreDispatchInterruptState ( DispatchCleanup & dispatch )

staticprivate

Definition at line 758 of file IrqHandlerRegistry.cc.

◆ retireSlot()

bool IrqHandlerRegistry::retireSlot	(	HandlerSlot &	slot,
		size_t	expectedPublication,
		IrqHandlerBase *	expectedHandler
	)

private

Definition at line 543 of file IrqHandlerRegistry.cc.

◆ retireSlotOrObserveClosed()

bool IrqHandlerRegistry::retireSlotOrObserveClosed	(	HandlerSlot &	slot,
		size_t	expectedPublication,
		IrqHandlerBase *	expectedHandler
	)

private

Definition at line 599 of file IrqHandlerRegistry.cc.

◆ snapshotLineConfiguration()

bool IrqHandlerRegistry::snapshotLineConfiguration	(	uint8_t	irq,
		LineConfiguration &	configuration
	)		const

Takes a bounded lock-free snapshot of one line's admitted configuration.

A false return means a concurrent mutation outlived both bounded attempts; callers may leave observational state stale and try again later.

Definition at line 1992 of file IrqHandlerRegistry.cc.

◆ threadedDispatchState()

size_t IrqHandlerRegistry::threadedDispatchState	(	uint8_t	irq,
		uintptr_t &	exactHandlerIdentity
	)		const

Lock-free debugger query for committed threaded callbacks on one line.

The committed pin includes the bounded setup and cleanup immediately around the virtual call. Returns the active count and writes the exact handler identity only when one pin is active. Multiple callbacks make the identity ambiguous and write zero.

Definition at line 2099 of file IrqHandlerRegistry.cc.

Referenced by Pic::snapshotIrqLines().

Here is the caller graph for this function:

◆ threadedGenerationValid()

bool IrqHandlerRegistry::threadedGenerationValid	(	uint8_t	irq,
		size_t	generation
	)		const

private

Definition at line 79 of file IrqHandlerRegistry.cc.

◆ tryReclaimTombstones()

void IrqHandlerRegistry::tryReclaimTombstones ( uint8_t irq )

private

Definition at line 498 of file IrqHandlerRegistry.cc.

◆ unpinActionMutation()

void IrqHandlerRegistry::unpinActionMutation	(	DispatchCleanup &	cleanup,
		Thread *	thread
	)

private

Definition at line 393 of file IrqHandlerRegistry.cc.

◆ unpublishDispatch()

bool IrqHandlerRegistry::unpublishDispatch	(	DispatchCleanup &	cleanup,
		bool	required
	)

private

Definition at line 662 of file IrqHandlerRegistry.cc.

◆ unregisterHandler() [1/2]

IrqHandlerRegistry::UnregisterResult IrqHandlerRegistry::unregisterHandler	(	uint8_t	irq,
		IrqHandlerBase *	handler
	)

Stops future callbacks and drains callbacks already in progress.

Self-removal cannot synchronously drain its own callback and is deferred until that callback returns. Atomic contexts reject a removal which would otherwise have to wait. A synchronous caller must not retain a resource which the active handler needs to finish.

The device source must be masked or quiesced before this call. Removal closes and drains registry admission; it cannot stop hardware from raising a new occurrence.

Definition at line 916 of file IrqHandlerRegistry.cc.

References unregisterHandler().

Referenced by VirtIrqManager::registerPciMsixIrqHandlers(), Pic::unregisterHandler(), VirtIrqManager::unregisterHandler(), and unregisterHandler().

Here is the caller graph for this function:

◆ unregisterHandler() [2/2]

IrqHandlerRegistry::UnregisterResult IrqHandlerRegistry::unregisterHandler	(	uint8_t	irq,
		IrqHandlerBase *	handler,
		LineMode &	removedDelivery
	)

Also reports the exact delivery of the slot whose admission was closed.

removedDelivery is Empty when no matching slot was found and is never Mixed: each registry slot retains one private delivery bit.

Definition at line 922 of file IrqHandlerRegistry.cc.

References Spinlock::acquire(), Thread::getDebugState(), ProcessorBase::getInterrupts(), ProcessorBase::information(), Scheduler::instance(), Spinlock::release(), Thread::setDebugState(), and Scheduler::yield().

Member Data Documentation

◆ DeliveryBits

constexpr size_t IrqHandlerRegistry::DeliveryBits = 1

staticconstexprprivate

Definition at line 318 of file IrqHandlerRegistry.h.

◆ DeliveryMask

constexpr size_t IrqHandlerRegistry::DeliveryMask = ((1 << DeliveryBits) - 1) << DeliveryShift

staticconstexprprivate

Definition at line 320 of file IrqHandlerRegistry.h.

◆ DeliveryShift

constexpr size_t IrqHandlerRegistry::DeliveryShift = ModeBits

staticconstexprprivate

Definition at line 319 of file IrqHandlerRegistry.h.

◆ GenerationShift

constexpr size_t IrqHandlerRegistry::GenerationShift = IrqShift + 8

staticconstexprprivate

Definition at line 323 of file IrqHandlerRegistry.h.

◆ GraceBucketCount

constexpr size_t IrqHandlerRegistry::GraceBucketCount = 16

staticconstexprprivate

Definition at line 294 of file IrqHandlerRegistry.h.

◆ InvalidIrq

constexpr uint8_t IrqHandlerRegistry::InvalidIrq = 0xFF

staticconstexprprivate

Definition at line 295 of file IrqHandlerRegistry.h.

◆ IrqCount

constexpr size_t IrqHandlerRegistry::IrqCount = 256

staticconstexprprivate

Definition at line 292 of file IrqHandlerRegistry.h.

◆ IrqMask

constexpr size_t IrqHandlerRegistry::IrqMask = 0xFF << IrqShift

staticconstexprprivate

Definition at line 322 of file IrqHandlerRegistry.h.

◆ IrqShift

constexpr size_t IrqHandlerRegistry::IrqShift = DeliveryShift + DeliveryBits

staticconstexprprivate

Definition at line 321 of file IrqHandlerRegistry.h.

◆ LineSnapshotAttempts

constexpr size_t IrqHandlerRegistry::LineSnapshotAttempts = 2

staticconstexprprivate

Definition at line 335 of file IrqHandlerRegistry.h.

◆ m_ActiveDispatches

ActiveDispatch IrqHandlerRegistry::m_ActiveDispatches[MaxActiveDispatches]

private

Definition at line 489 of file IrqHandlerRegistry.h.

◆ m_AdmissionEpoch

size_t IrqHandlerRegistry::m_AdmissionEpoch

private

Definition at line 499 of file IrqHandlerRegistry.h.

◆ m_HandlerLock

Spinlock IrqHandlerRegistry::m_HandlerLock

private

Definition at line 498 of file IrqHandlerRegistry.h.

◆ m_Handlers

HandlerSlot IrqHandlerRegistry::m_Handlers[MaxHandlerSlots]

private

Definition at line 488 of file IrqHandlerRegistry.h.

◆ m_HardHandoffEpochs

size_t IrqHandlerRegistry::m_HardHandoffEpochs[GraceBucketCount]

private

Makes the bounded per-line hard-veto scan coherent across mutations.

Definition at line 491 of file IrqHandlerRegistry.h.

Referenced by hardLineQuarantined().

◆ m_MutationGeneration

size_t IrqHandlerRegistry::m_MutationGeneration

private

Definition at line 500 of file IrqHandlerRegistry.h.

◆ m_MutationWriters

size_t IrqHandlerRegistry::m_MutationWriters

private

Definition at line 501 of file IrqHandlerRegistry.h.

◆ m_OccurrenceBoundaryLocks

size_t IrqHandlerRegistry::m_OccurrenceBoundaryLocks[GraceBucketCount]

private

Definition at line 497 of file IrqHandlerRegistry.h.

◆ m_OccurrenceEpochs

size_t IrqHandlerRegistry::m_OccurrenceEpochs[GraceBucketCount]

private

Definition at line 495 of file IrqHandlerRegistry.h.

◆ m_OccurrenceReaders

size_t IrqHandlerRegistry::m_OccurrenceReaders[GraceBucketCount][2]

private

Definition at line 496 of file IrqHandlerRegistry.h.

◆ m_ThreadedActionMutationGeneration

size_t IrqHandlerRegistry::m_ThreadedActionMutationGeneration

private

Definition at line 493 of file IrqHandlerRegistry.h.

◆ m_ThreadedActionMutationWriters

size_t IrqHandlerRegistry::m_ThreadedActionMutationWriters

private

Definition at line 494 of file IrqHandlerRegistry.h.

◆ m_ThreadedInvalidationGenerations

size_t IrqHandlerRegistry::m_ThreadedInvalidationGenerations[IrqCount]

private

Definition at line 492 of file IrqHandlerRegistry.h.

◆ MaxActiveDispatches

constexpr size_t IrqHandlerRegistry::MaxActiveDispatches = 64

staticconstexprprivate

Definition at line 291 of file IrqHandlerRegistry.h.

◆ MaxHandlerSlots

constexpr size_t IrqHandlerRegistry::MaxHandlerSlots = 64

staticconstexprprivate

Definition at line 290 of file IrqHandlerRegistry.h.

◆ MaximumPublicationGeneration

constexpr size_t IrqHandlerRegistry::MaximumPublicationGeneration = ~static_cast<size_t>(0) >> GenerationShift

staticconstexprprivate

Definition at line 324 of file IrqHandlerRegistry.h.

◆ ModeBits

constexpr size_t IrqHandlerRegistry::ModeBits = 3

staticconstexprprivate

Definition at line 316 of file IrqHandlerRegistry.h.

◆ ModeMask

constexpr size_t IrqHandlerRegistry::ModeMask = (1 << ModeBits) - 1

staticconstexprprivate

Definition at line 317 of file IrqHandlerRegistry.h.

◆ PolicyControllerAckMask

constexpr size_t IrqHandlerRegistry::PolicyControllerAckMask = 3U << PolicyControllerAckShift

staticconstexprprivate

Definition at line 331 of file IrqHandlerRegistry.h.

◆ PolicyControllerAckShift

constexpr size_t IrqHandlerRegistry::PolicyControllerAckShift = 3

staticconstexprprivate

Definition at line 328 of file IrqHandlerRegistry.h.

◆ PolicyLineReleaseMask

constexpr size_t IrqHandlerRegistry::PolicyLineReleaseMask = 1U << PolicyLineReleaseShift

staticconstexprprivate

Definition at line 332 of file IrqHandlerRegistry.h.

◆ PolicyLineReleaseShift

constexpr size_t IrqHandlerRegistry::PolicyLineReleaseShift = 5

staticconstexprprivate

Definition at line 329 of file IrqHandlerRegistry.h.

◆ PolicyMixedCompatibilityMask

constexpr size_t IrqHandlerRegistry::PolicyMixedCompatibilityMask

staticconstexprprivate

Initial value:

=

PolicyTriggerMask | PolicyControllerAckMask

Definition at line 333 of file IrqHandlerRegistry.h.

◆ PolicyTriggerMask

constexpr size_t IrqHandlerRegistry::PolicyTriggerMask = 3U << PolicyTriggerShift

staticconstexprprivate

Definition at line 330 of file IrqHandlerRegistry.h.

◆ PolicyTriggerShift

constexpr size_t IrqHandlerRegistry::PolicyTriggerShift = 1

staticconstexprprivate

Definition at line 327 of file IrqHandlerRegistry.h.

◆ PolicyValid

constexpr size_t IrqHandlerRegistry::PolicyValid = 1U << 0

staticconstexprprivate

Definition at line 326 of file IrqHandlerRegistry.h.

◆ QuiescedLaneCount

constexpr size_t IrqHandlerRegistry::QuiescedLaneCount = static_cast<size_t>(QuiescedLane::Count)

staticconstexprprivate

Definition at line 314 of file IrqHandlerRegistry.h.

The documentation for this class was generated from the following files:

Classes

Public Types

Public Member Functions

Private Types

Private Member Functions

Static Private Member Functions

Private Attributes

Static Private Attributes

Detailed Description

Member Enumeration Documentation

◆ Delivery

◆ LineMode

◆ QuiescedLane

◆ SlotMode

◆ UnregisterResult

Constructor & Destructor Documentation

◆ IrqHandlerRegistry()

Member Function Documentation

◆ abandonAdmissionCutoff()

◆ abandonDispatch()

◆ abandonThreadedActionMutation()

◆ acquireFinalizationGate()

◆ acquireOccurrenceReaderLeases()

◆ beginAdmissionCutoffCleanup()

◆ beginMutation()

◆ beginThreadedActionMutation()

◆ canWaitForActionFinalization()

◆ captureAdmissionCutoff()

◆ captureMixedAdmissionCutoffs()

◆ closeSlotAdmission()

◆ completeThreadedActionMutation()

◆ containsHandler()

◆ currentDispatchOwner()

◆ decodePolicy()

◆ deliveryOf()

◆ dispatchHard() [1/2]

◆ dispatchHard() [2/2]

◆ dispatchThreaded()

◆ effectiveMixedPolicy()

◆ encodePolicy()

◆ findCurrentDispatch()

◆ finishAdmissionCutoffCleanup()

◆ finishMutation()

◆ finishThreadedActionMutation()

◆ generationOf()

◆ generationReached()

◆ handlerCount()

◆ hardDispatchState()

◆ hardLineQuarantined()

◆ hasActiveDispatch()

◆ hasQuiescedGeneration()

◆ invalidateThreadedGenerationFromInterrupt()

◆ invalidateThreadedLine()

◆ irqOf()

◆ lineMode()

◆ lineModeForDelivery()

◆ makePublication()

◆ mixedPoliciesCompatible()

◆ modeOf()

◆ occurrencePrecedesRetirement()

◆ pinActionMutation()

◆ publishDispatch()

◆ publishSlotQuiesced()

◆ publishSlotQuiescedValue()

◆ publishThreadedDispatch() [1/2]

◆ publishThreadedDispatch() [2/2]

◆ quiescedLane() [1/2]

◆ quiescedLane() [2/2]

◆ registerHandler()

◆ registerHardHandler() [1/2]

◆ registerHardHandler() [2/2]

◆ registerThreadedHandler() [1/2]

◆ registerThreadedHandler() [2/2]

◆ releaseAdmissionCutoff()

◆ releaseFinalizationGate()

◆ releaseOccurrenceReaderLeases()

◆ restoreDispatchInterruptState()

◆ retireSlot()

◆ retireSlotOrObserveClosed()

◆ snapshotLineConfiguration()