The Pedigree Project 0.1
IrqHandlerRegistry.h
1/*
2 * Copyright (c) 2026, Pedigree Developers
3 *
4 * Permission to use, copy, modify, and distribute this software for any
5 * purpose with or without fee is hereby granted.
6 */
7
8#ifndef PEDIGREE_KERNEL_MACHINE_IRQHANDLERREGISTRY_H
9#define PEDIGREE_KERNEL_MACHINE_IRQHANDLERREGISTRY_H
10#include "pedigree/kernel/Spinlock.h"
11#include "pedigree/kernel/compiler.h"
12#include "pedigree/kernel/machine/IrqManager.h"
13#include "pedigree/kernel/process/AtomicStateCleanup.h"
14#include "pedigree/kernel/processor/state_forward.h"
15#include "pedigree/kernel/processor/types.h"
16
17#include <config.h>
18
19class HardIrqHandler;
20class IrqHandler;
21class IrqHandlerBase;
22class Thread;
23enum class HardIrqDisposition;
24
35class EXPORTED_PUBLIC IrqHandlerRegistry {
36 public:
37 struct ThreadedDispatchResult {
38 bool handled;
39 bool allowRearm;
40 };
41
42 struct AdmissionCutoff {
43 size_t epoch;
44 size_t occurrenceEpoch;
45 size_t readerToken;
46 };
47
48 struct MixedAdmissionCutoffs {
49 AdmissionCutoff hard;
50 AdmissionCutoff threaded;
51 };
52
53 enum class UnregisterResult {
54 Completed,
55 Deferred,
56 Rejected,
57 NotFound,
58 };
59
60 enum class LineMode {
61 Empty,
62 Threaded,
63 HardOnly,
64 Mixed,
65 };
66
68 struct LineConfiguration {
69 LineConfiguration()
70 : mutationGeneration(0),
71 handlerCount(0),
72 mode(LineMode::Empty),
73 policyConfigured(false),
74 trigger(IrqTrigger::Edge),
75 controllerAck(IrqControllerAck::None),
76 lineRelease(IrqLineRelease::AfterHardStage) {}
77
78 size_t mutationGeneration;
79 size_t handlerCount;
80 LineMode mode;
81 bool policyConfigured;
82 IrqTrigger trigger;
83 IrqControllerAck controllerAck;
84 IrqLineRelease lineRelease;
85 };
86
87 IrqHandlerRegistry();
88
90 bool registerThreadedHandler(uint8_t irq, IrqHandler* handler);
91
93 bool registerThreadedHandler(uint8_t irq, IrqHandler* handler, const IrqPolicy& policy);
94
96 bool registerHardHandler(uint8_t irq, HardIrqHandler* handler);
97
99 bool registerHardHandler(uint8_t irq, HardIrqHandler* handler, const IrqPolicy& policy);
100
113 UnregisterResult unregisterHandler(uint8_t irq, IrqHandlerBase* handler);
114
121 UnregisterResult unregisterHandler(uint8_t irq, IrqHandlerBase* handler,
122 LineMode& removedDelivery);
123
131 bool dispatchHard(uint8_t irq, InterruptState& state, HardIrqDisposition& disposition,
132 HardIrqHandler* onlyHandler = nullptr, size_t dispatchGeneration = 0,
133 irq_id_t callbackId = 0);
134 bool dispatchHard(uint8_t irq, InterruptState& state, HardIrqDisposition& disposition,
135 HardIrqHandler* onlyHandler, size_t dispatchGeneration,
136 AdmissionCutoff admissionCutoff, irq_id_t callbackId = 0);
137
139 bool hardLineQuarantined(uint8_t irq) const;
140
151 bool publishThreadedDispatch(uint8_t irq, size_t dispatchGeneration);
152 bool publishThreadedDispatch(uint8_t irq, size_t dispatchGeneration,
153 AdmissionCutoff admissionCutoff);
154
159 bool captureAdmissionCutoff(uint8_t irq, AdmissionCutoff& cutoff);
160
168 bool captureMixedAdmissionCutoffs(uint8_t irq, MixedAdmissionCutoffs& cutoffs);
169
171 void releaseAdmissionCutoff(AdmissionCutoff admissionCutoff);
172
174 void invalidateThreadedLine(uint8_t irq, size_t throughGeneration);
175
183 void invalidateThreadedGenerationFromInterrupt(uint8_t irq, size_t throughGeneration);
184
195 bool dispatchThreaded(uint8_t irq, size_t dispatchGeneration, ThreadedDispatchResult& result,
196 IrqHandler* onlyHandler = nullptr, irq_id_t callbackId = 0);
197
198 size_t handlerCount(uint8_t irq);
199
201 bool containsHandler(uint8_t irq, IrqHandlerBase* handler) const;
202
204 LineMode lineMode(uint8_t irq);
205
213 bool snapshotLineConfiguration(uint8_t irq, LineConfiguration& configuration) const;
214
222 size_t hardDispatchState(uint8_t irq, size_t& exactGeneration) const;
223
232 size_t threadedDispatchState(uint8_t irq, uintptr_t& exactHandlerIdentity) const;
233
234#if HOSTED && PEDIGREE_HOSTED_SMOKE_TESTS
235 enum class HandlerHazardStage {
236 BeforeClaim,
237 Claimed,
238 Committed,
239 Released,
240 RetirementBoundaryPublished,
241 BeforeClaimFinalization,
242 FinalizationContended,
243 BeforeActionMutationPin,
244 QuiescedObserved,
245 BeforePendingExchange,
246 PendingExchanged,
247 BeforeQuiescedExchange,
248 QuiescedExchanged,
249 };
250
251 enum class OccurrenceCaptureStage {
252 BankZeroClaimed,
253 BankOneClaimed,
254 EpochSampled,
255 UnusedBankReleased,
256 };
257
258 using HandlerPinHook = void (*)(IrqHandlerBase*);
259 using HandlerPrePinHook = void (*)(IrqHandlerBase*);
260 using HandlerHazardHook = void (*)(IrqHandlerBase*, HandlerHazardStage);
261 using DispatchAbandonHook = void (*)(void*, bool);
262 using MutationLockHook = void (*)();
263 using OccurrenceCaptureHook = void (*)(IrqHandlerRegistry*, uint8_t, OccurrenceCaptureStage,
264 size_t);
265
266 void setHandlerPinHook(HandlerPinHook hook);
267 void setHandlerPrePinHook(HandlerPrePinHook hook);
268 void setHandlerHazardHook(HandlerHazardHook hook);
269 void setDispatchAbandonHook(DispatchAbandonHook hook);
270 void setOccurrenceCaptureHookForTest(OccurrenceCaptureHook hook);
271 void withMutationLockForTest(MutationLockHook hook);
272 void withMutationEpochForTest(MutationLockHook hook);
273 size_t activeDispatchCountForTest(IrqHandlerBase* handler);
274 size_t claimedDispatchCountForOwnerForTest(void* owner);
275 bool containsHandlerForTest(uint8_t irq, IrqHandlerBase* handler);
276 size_t tombstoneCountForTest(uint8_t irq) const;
277 size_t threadedActionMutationWriterCountForTest() const;
278 bool setThreadedActionLanesForTest(IrqHandlerBase* handler, size_t pendingGeneration,
279 size_t claimedGeneration, size_t quiescedGeneration);
280 bool consumeThreadedQuiescedForTest(IrqHandlerBase* handler, size_t generation);
281 bool publishControllerQuiescedForTest(IrqHandlerBase* handler, uint8_t irq, size_t generation);
282#endif
283
284 private:
285 enum class Delivery : size_t {
286 Threaded,
287 HardOnly,
288 };
289
290 static constexpr size_t MaxHandlerSlots = 64;
291 static constexpr size_t MaxActiveDispatches = 64;
292 static constexpr size_t IrqCount = 256;
293 // Bucket collisions delay tombstone reuse but cannot admit a stale slot.
294 static constexpr size_t GraceBucketCount = 16;
295 static constexpr uint8_t InvalidIrq = 0xFF;
296
297 enum class SlotMode : size_t {
298 Empty = 0,
299 Enabled,
300 Draining,
301 Cancelling,
302 Closed,
303 Retiring,
304 Tombstone,
305 };
306
307 enum class QuiescedLane : size_t {
308 Controller,
309 Callback,
310 Retirement,
311 Count,
312 };
313
314 static constexpr size_t QuiescedLaneCount = static_cast<size_t>(QuiescedLane::Count);
315
316 static constexpr size_t ModeBits = 3;
317 static constexpr size_t ModeMask = (1 << ModeBits) - 1;
318 static constexpr size_t DeliveryBits = 1;
319 static constexpr size_t DeliveryShift = ModeBits;
320 static constexpr size_t DeliveryMask = ((1 << DeliveryBits) - 1) << DeliveryShift;
321 static constexpr size_t IrqShift = DeliveryShift + DeliveryBits;
322 static constexpr size_t IrqMask = 0xFF << IrqShift;
323 static constexpr size_t GenerationShift = IrqShift + 8;
324 static constexpr size_t MaximumPublicationGeneration = ~static_cast<size_t>(0) >> GenerationShift;
325
326 static constexpr size_t PolicyValid = 1U << 0;
327 static constexpr size_t PolicyTriggerShift = 1;
328 static constexpr size_t PolicyControllerAckShift = 3;
329 static constexpr size_t PolicyLineReleaseShift = 5;
330 static constexpr size_t PolicyTriggerMask = 3U << PolicyTriggerShift;
331 static constexpr size_t PolicyControllerAckMask = 3U << PolicyControllerAckShift;
332 static constexpr size_t PolicyLineReleaseMask = 1U << PolicyLineReleaseShift;
333 static constexpr size_t PolicyMixedCompatibilityMask =
334 PolicyTriggerMask | PolicyControllerAckMask;
335 static constexpr size_t LineSnapshotAttempts = 2;
336
337 struct HandlerSlot;
338
339 struct ActiveDispatch {
340 ActiveDispatch()
341 : token(nullptr),
342 generation(0),
343 owner(nullptr),
344 admittedPublication(0),
345 controllerGeneration(0),
346 slot(nullptr),
347 callback(0) {}
348
349 void* token;
350 size_t generation;
351 void* owner;
352 size_t admittedPublication;
353 size_t controllerGeneration;
354 HandlerSlot* slot;
355 size_t callback;
356 };
357
358 struct HandlerSlot {
359 HandlerSlot()
360 : handler(nullptr),
361 publication(0),
362 policy(0),
363 admissionEpoch(0),
364 pendingThreadedGeneration(0),
365 claimedThreadedGeneration(0),
366 quiescedThreadedGenerations(),
367 retirementEpoch(0),
368 hardHandoffState(0),
369 finalizationGate(0) {}
370
371 IrqHandlerBase* handler;
372 size_t publication;
373 size_t policy;
374 size_t admissionEpoch;
375 size_t pendingThreadedGeneration;
376 size_t claimedThreadedGeneration;
377 size_t quiescedThreadedGenerations[QuiescedLaneCount];
378 size_t retirementEpoch;
380 size_t hardHandoffState;
381 size_t finalizationGate;
382 };
383
384 struct DispatchCleanup {
385 DispatchCleanup(IrqHandlerRegistry* registry, HandlerSlot* handlerSlot, void* dispatchOwner,
386 size_t admittedPublication, bool isCallback = true)
387 : registry(registry),
388 slot(handlerSlot),
389 owner(dispatchOwner),
390 publication(admittedPublication),
391 callback(isCallback),
392 activeDispatch(nullptr),
393 previousDeviceHardIrqDepth(0),
394 restoreDeviceHardIrqDepth(false),
395 previousInterruptState(false),
396 restoreInterruptState(false),
397 cleanup() {}
398
399 IrqHandlerRegistry* registry;
400 HandlerSlot* slot;
401 void* owner;
402 size_t publication;
403 bool callback;
404 ActiveDispatch* activeDispatch;
405 size_t previousDeviceHardIrqDepth;
406 bool restoreDeviceHardIrqDepth;
407 bool previousInterruptState;
408 bool restoreInterruptState;
409 AtomicStateCleanupRecord cleanup;
410 };
411
412 struct AdmissionCutoffCleanup {
413 AdmissionCutoffCleanup(IrqHandlerRegistry* owner, AdmissionCutoff admissionCutoff)
414 : registry(owner), cutoff(admissionCutoff), thread(nullptr), ownsCutoff(false), cleanup() {}
415
416 IrqHandlerRegistry* registry;
417 AdmissionCutoff cutoff;
418 Thread* thread;
419 bool ownsCutoff;
420 AtomicStateCleanupRecord cleanup;
421 };
422
423 struct ThreadedActionMutationCleanup {
424 explicit ThreadedActionMutationCleanup(IrqHandlerRegistry* owner)
425 : registry(owner), thread(nullptr), cleanup() {}
426
427 IrqHandlerRegistry* registry;
428 Thread* thread;
429 AtomicStateCleanupRecord cleanup;
430 };
431
432 static size_t makePublication(size_t generation, uint8_t irq, SlotMode mode, Delivery delivery);
433 static size_t generationOf(size_t publication);
434 static uint8_t irqOf(size_t publication);
435 static SlotMode modeOf(size_t publication);
436 static Delivery deliveryOf(size_t publication);
437 static bool generationReached(size_t current, size_t target);
438 static size_t encodePolicy(const IrqPolicy* policy);
439 static void decodePolicy(size_t policy, LineConfiguration& configuration);
440 static bool mixedPoliciesCompatible(size_t first, size_t second);
441 static size_t effectiveMixedPolicy(size_t hard, size_t threaded);
442 static LineMode lineModeForDelivery(Delivery delivery);
443
444 bool registerHandler(uint8_t irq, IrqHandlerBase* handler, Delivery delivery, size_t policy);
445
446 bool acquireOccurrenceReaderLeases(uint8_t irq, size_t readerBank, size_t count);
447 void releaseOccurrenceReaderLeases(uint8_t irq, size_t readerBank, size_t count);
448 void beginAdmissionCutoffCleanup(AdmissionCutoffCleanup& cleanup);
449 void finishAdmissionCutoffCleanup(AdmissionCutoffCleanup& cleanup);
450 static void abandonAdmissionCutoff(void* context);
451
452 void beginMutation();
453 void finishMutation();
454 void beginThreadedActionMutation(ThreadedActionMutationCleanup& cleanup);
455 void finishThreadedActionMutation(ThreadedActionMutationCleanup& cleanup);
456 void completeThreadedActionMutation();
457 static void abandonThreadedActionMutation(void* context);
458
459 bool retireSlot(HandlerSlot& slot, size_t expectedPublication, IrqHandlerBase* expectedHandler);
460 bool retireSlotOrObserveClosed(HandlerSlot& slot, size_t expectedPublication,
461 IrqHandlerBase* expectedHandler);
462 bool closeSlotAdmission(HandlerSlot& slot, size_t expectedPublication, size_t& closedPublication);
463 void tryReclaimTombstones(uint8_t irq);
464 bool occurrencePrecedesRetirement(const HandlerSlot& slot, AdmissionCutoff admissionCutoff) const;
465 bool publishDispatch(DispatchCleanup& cleanup, size_t controllerGeneration);
466 bool unpublishDispatch(DispatchCleanup& cleanup, bool required);
467 static void abandonDispatch(void* context);
468 static void restoreDispatchInterruptState(DispatchCleanup& dispatch);
469 static bool canWaitForActionFinalization();
470 bool acquireFinalizationGate(HandlerSlot& slot, bool canWait);
471 static void releaseFinalizationGate(HandlerSlot& slot);
472 bool pinActionMutation(HandlerSlot& slot, size_t publication, DispatchCleanup& cleanup,
473 Thread* thread);
474 void unpinActionMutation(DispatchCleanup& cleanup, Thread* thread);
475 bool hasActiveDispatch(HandlerSlot& slot, size_t admittedPublication) const;
476 bool findCurrentDispatch(void* owner, HandlerSlot* target, size_t targetPublication,
477 bool& callbackContext) const;
478 static void* currentDispatchOwner();
479 bool threadedGenerationValid(uint8_t irq, size_t generation) const;
480 void publishSlotQuiesced(HandlerSlot& slot, uint8_t irq, size_t dispatchGeneration,
481 QuiescedLane lane);
482 void publishSlotQuiescedValue(HandlerSlot& slot, uint8_t irq, size_t dispatchGeneration,
483 QuiescedLane lane);
484 static size_t* quiescedLane(HandlerSlot& slot, QuiescedLane lane);
485 static const size_t* quiescedLane(const HandlerSlot& slot, QuiescedLane lane);
486 static bool hasQuiescedGeneration(const HandlerSlot& slot);
487
488 HandlerSlot m_Handlers[MaxHandlerSlots];
489 ActiveDispatch m_ActiveDispatches[MaxActiveDispatches];
491 size_t m_HardHandoffEpochs[GraceBucketCount];
492 size_t m_ThreadedInvalidationGenerations[IrqCount];
493 size_t m_ThreadedActionMutationGeneration;
494 size_t m_ThreadedActionMutationWriters;
495 size_t m_OccurrenceEpochs[GraceBucketCount];
496 size_t m_OccurrenceReaders[GraceBucketCount][2];
497 size_t m_OccurrenceBoundaryLocks[GraceBucketCount];
498 Spinlock m_HandlerLock;
499 size_t m_AdmissionEpoch;
500 size_t m_MutationGeneration;
501 size_t m_MutationWriters;
502
503#if HOSTED && PEDIGREE_HOSTED_SMOKE_TESTS
504 void observeOccurrenceCaptureForTest(uint8_t irq, OccurrenceCaptureStage stage,
505 size_t occurrenceEpoch);
506 HandlerPinHook m_HandlerPinHook;
507 HandlerPrePinHook m_HandlerPrePinHook;
508 HandlerHazardHook m_HandlerHazardHook;
509 DispatchAbandonHook m_DispatchAbandonHook;
510 OccurrenceCaptureHook m_OccurrenceCaptureHook;
511#endif
512};
513
514#endif
HardIrqDisposition
HardIrqDisposition
Definition IrqHandler.h:44
IrqControllerAck
IrqControllerAck
Definition include/pedigree/kernel/machine/IrqManager.h:47
IrqLineRelease
IrqLineRelease
Definition include/pedigree/kernel/machine/IrqManager.h:54
IrqTrigger
IrqTrigger
Definition include/pedigree/kernel/machine/IrqManager.h:40
Generated on Sat Oct 3 2026 06:38:07 for The Pedigree Project by doxygen 1.9.8