8#include "pedigree/kernel/Atomic.h"
9#include "pedigree/kernel/Log.h"
10#include "pedigree/kernel/machine/Machine.h"
11#include "pedigree/kernel/machine/Timer.h"
12#include "pedigree/kernel/machine/TimerHandler.h"
13#include "pedigree/kernel/machine/TimerHandlerRegistry.h"
14#include "pedigree/kernel/process/Event.h"
15#include "pedigree/kernel/process/PerProcessorScheduler.h"
16#include "pedigree/kernel/process/RelayEvent.h"
17#include "pedigree/kernel/process/Scheduler.h"
18#include "pedigree/kernel/process/Semaphore.h"
19#include "pedigree/kernel/process/Thread.h"
20#include "pedigree/kernel/process/eventNumbers.h"
21#include "pedigree/kernel/processor/Processor.h"
22#include "pedigree/kernel/time/Time.h"
24#include "system/kernel/machine/hosted/Timer.h"
39Event* g_DispositionAEvent =
nullptr;
40Event* g_DispositionBEvent =
nullptr;
42bool check(
bool condition,
const char* test,
const char* detail);
46 uintptr_t debugAddress = 0;
47 return thread && thread->
getWaitDebugInfo(wait) && wait.queue && wait.channelOwner &&
48 wait.channelValue && wait.queued && wait.reason == WaitQueue::WakeReason::Waiting &&
49 thread->
getDebugState(debugAddress) == Thread::CallbackDrain &&
50 debugAddress ==
reinterpret_cast<uintptr_t
>(handler);
53struct RegistryDispatchContext;
54RegistryDispatchContext* g_TimerRegistryDispatchContext =
nullptr;
55void dispatchTimerWhileWriterLocked();
57class RegistryDispatchTimerHandler :
public TimerHandler {
59 explicit RegistryDispatchTimerHandler(RegistryDispatchContext& context) : m_Context(context) {}
61 void timer(uint64_t delta)
override;
64 RegistryDispatchContext& m_Context;
67struct RegistryDispatchContext {
68 explicit RegistryDispatchContext(
Timer* timer)
74 unregisterSucceeded(0),
79 RegistryDispatchTimerHandler handler;
88void RegistryDispatchTimerHandler::timer(uint64_t delta) {
90 if (m_Context.mutationRequested.compareAndSwap(1, 2)) {
91 m_Context.delta = delta;
92 HostedTimer::withHandlerMutationLockForTest(dispatchTimerWhileWriterLocked);
93 m_Context.mutationRequested = 3;
97void dispatchTimerWhileWriterLocked() {
98 RegistryDispatchContext* context = g_TimerRegistryDispatchContext;
103 context->hookCalls += 1;
104 if (HostedTimer::dispatchHandlerForTest(&context->handler, context->delta)) {
105 context->admitted += 1;
109struct AtomicDrainRaceContext;
110AtomicDrainRaceContext* g_AtomicDrainRaceContext =
nullptr;
114 explicit AtomicDrainRaceHandler(AtomicDrainRaceContext& context) : m_Context(context) {}
116 void timer(uint64_t)
override;
119 AtomicDrainRaceContext& m_Context;
122struct AtomicDrainRaceContext {
123 explicit AtomicDrainRaceContext(
Timer* timer)
126 removerReady(0, false),
127 beginDrain(0, false),
132 selfRemovalRejected(0),
134 atomicRemovalRejected(0),
135 beginDrainTimedOut(0),
136 pinTransitionTimedOut(0),
137 drainCompletionTimedOut(0),
141 AtomicDrainRaceHandler handler;
157void AtomicDrainRaceHandler::timer(uint64_t) {
158 m_Context.handlerCalls += 1;
159 if (m_Context.phase !=
static_cast<size_t>(2)) {
163 if (!m_Context.timer->unregisterHandler(
this)) {
164 m_Context.selfRemovalRejected += 1;
166 if (m_Context.timer->registerHandler(
this)) {
167 m_Context.revivalSucceeded += 1;
173 constexpr size_t YieldLimit = 10000;
174 AtomicDrainRaceContext* context = g_AtomicDrainRaceContext;
175 if (!context || handler != &context->handler || !context->phase.compareAndSwap(0, 1)) {
179 context->pinHookCalls += 1;
180 context->beginDrain.release();
181 for (
size_t i = 0; context->phase !=
static_cast<size_t>(2) && i < YieldLimit; ++i) {
184 if (context->phase !=
static_cast<size_t>(2)) {
185 context->pinTransitionTimedOut += 1;
186 context->failures += 1;
192 constexpr size_t YieldLimit = 10000;
193 AtomicDrainRaceContext* context = g_AtomicDrainRaceContext;
194 if (!context || handler != &context->handler || !context->phase.compareAndSwap(1, 2)) {
195 if (context && handler == &context->handler) {
196 context->failures += 1;
201 context->drainHookCalls += 1;
202 for (
size_t i = 0; context->phase !=
static_cast<size_t>(3) && i < YieldLimit; ++i) {
205 if (context->phase !=
static_cast<size_t>(3)) {
206 context->drainCompletionTimedOut += 1;
207 context->failures += 1;
212int removeTimerAtomically(
void* parameter) {
213 AtomicDrainRaceContext* context =
reinterpret_cast<AtomicDrainRaceContext*
>(parameter);
214 context->removerReady.release();
215 if (!context->beginDrain.acquireForCompletion(1, 2)) {
216 context->beginDrainTimedOut += 1;
217 context->failures += 1;
220 if (context->phase !=
static_cast<size_t>(1)) {
221 context->failures += 1;
227 const bool removed = context->timer->unregisterHandler(&context->handler);
230 context->atomicRemovalRejected += 1;
232 context->failures += 1;
237enum class TimerAbandonPoint {
243struct TimerAbandonContext;
244TimerAbandonContext* g_TimerAbandonContext =
nullptr;
248 explicit AbandoningTimerHandler(TimerAbandonContext& context) : m_Context(context) {}
250 void timer(uint64_t)
override;
253 TimerAbandonContext& m_Context;
256struct TimerAbandonContext {
266 hookObservedDrain(0),
269 unregisterReturned(0),
270 unregisterSucceeded(0),
271 selfRemovalRejected(0),
276 AbandoningTimerHandler handler;
279 TimerAbandonPoint point;
291void abandonCurrentTimerDispatcher() {
293 PerProcessorScheduler::StackDiscardReason::HostedRegression);
296void AbandoningTimerHandler::timer(uint64_t) {
301 m_Context.handlerCalls += 1;
302 if (m_Context.point != TimerAbandonPoint::DeferredSelfRemoval) {
303 m_Context.failures += 1;
307 if (!m_Context.registry->unregisterHandler(
this)) {
308 m_Context.selfRemovalRejected += 1;
310 m_Context.failures += 1;
312 abandonCurrentTimerDispatcher();
316 TimerAbandonContext* context = g_TimerAbandonContext;
317 if (!context || handler != &context->handler ||
318 context->point != TimerAbandonPoint::PartialHazard ||
323 context->hookCalls += 1;
324 abandonCurrentTimerDispatcher();
327void abandonCommittedTimerHazard(
TimerHandler* handler) {
328 TimerAbandonContext* context = g_TimerAbandonContext;
329 if (!context || handler != &context->handler ||
330 context->point != TimerAbandonPoint::CommittedHazard ||
332 !context->phase.compareAndSwap(0, 1)) {
336 context->hookCalls += 1;
337 const uint64_t deadline =
338 context->timer->getTickCountNano() + (500 * Time::Multiplier::Millisecond);
339 while (context->timer->getTickCountNano() < deadline) {
340 if (context->phase ==
static_cast<size_t>(2) && context->remover &&
341 hasTimerCallbackDrainWait(context->remover, &context->handler)) {
342 context->hookObservedDrain += 1;
343 abandonCurrentTimerDispatcher();
348 context->failures += 1;
349 abandonCurrentTimerDispatcher();
352int awaitAbandoningTimer(
void* parameter) {
353 TimerAbandonContext* context =
reinterpret_cast<TimerAbandonContext*
>(parameter);
358 context->registry->dispatch(Time::Multiplier::Millisecond, &context->handler);
359 context->failures += 1;
360 context->dispatchReturned += 1;
364int unregisterAbandoningTimer(
void* parameter) {
365 TimerAbandonContext* context =
reinterpret_cast<TimerAbandonContext*
>(parameter);
366 const uint64_t deadline =
367 context->timer->getTickCountNano() + (500 * Time::Multiplier::Millisecond);
368 while (context->phase !=
static_cast<size_t>(1) &&
369 context->timer->getTickCountNano() < deadline) {
372 if (context->phase !=
static_cast<size_t>(1)) {
373 context->failures += 1;
378 if (context->registry->unregisterHandler(&context->handler)) {
379 context->unregisterSucceeded += 1;
381 context->unregisterReturned += 1;
385bool timerPartialHazardAbandonment() {
386 constexpr const char* Test =
"timer-partial-hazard-abandonment";
389 TimerAbandonContext context(timer, ®istry, TimerAbandonPoint::PartialHazard);
391 &context,
nullptr,
false,
true,
true);
392 context.dispatcher->setName(
"hosted partial timer hazard");
394 g_TimerAbandonContext = &context;
395 registry.setHandlerHazardClaimHook(abandonPartialTimerHazard);
396 const bool registered = registry.registerHandler(&context.handler);
397 const bool started = registered && context.dispatcher->start();
398 const bool joined = started && context.dispatcher->join();
399 registry.setHandlerHazardClaimHook(
nullptr);
400 g_TimerAbandonContext =
nullptr;
402 const size_t claimed = registry.claimedDispatchCountForTest();
403 const size_t active = registry.activeDispatchCountForTest(&context.handler);
405 const bool reused = removed && registry.registerHandler(&context.handler);
409 passed &= check(registered && started && joined, Test,
410 "the partial-publication worker did not complete");
411 passed &= check(context.hookCalls == 1 && context.handlerCalls == 0 &&
412 context.dispatchReturned == 0 && context.failures == 0,
413 Test,
"dispatch escaped the forced partial-publication window");
414 passed &= check(claimed == 0 && active == 0, Test,
415 "stack abandonment leaked a partial callback hazard");
416 passed &= check(removed && reused && cleaned, Test,
417 "the handler slot could not be removed and reused after cleanup");
419 NOTICE(
"HOSTED-WAIT-TEST: PASS timer-partial-hazard-abandonment");
424bool timerCommittedHazardAbandonment() {
425 constexpr const char* Test =
"timer-abandoned-dispatch-cleanup";
428 TimerAbandonContext context(timer, ®istry, TimerAbandonPoint::CommittedHazard);
430 &context,
nullptr,
false,
true);
431 context.remover->setName(
"hosted abandoned timer remover");
433 &context,
nullptr,
false,
true,
true);
434 context.dispatcher->setName(
"hosted abandoned timer dispatch");
436 g_TimerAbandonContext = &context;
437 registry.setHandlerPinHook(abandonCommittedTimerHazard);
438 const bool registered = registry.registerHandler(&context.handler);
442 const bool started = registered && context.dispatcher->start();
443 const bool dispatcherJoined = started && context.dispatcher->join();
444 const bool removerJoined = context.remover->join();
445 registry.setHandlerPinHook(
nullptr);
446 g_TimerAbandonContext =
nullptr;
448 if (!context.unregisterSucceeded && registered) {
451 const size_t claimed = registry.claimedDispatchCountForTest();
452 const size_t active = registry.activeDispatchCountForTest(&context.handler);
453 const bool reused = context.unregisterSucceeded && registry.registerHandler(&context.handler);
457 passed &= check(registered && started && dispatcherJoined && removerJoined, Test,
458 "the abandoned-dispatch workers did not complete");
459 passed &= check(context.failures == 0 && context.hookCalls == 1 && context.hookObservedDrain == 1,
460 Test,
"the committed drain-and-abandon window was not reached");
461 passed &= check(context.handlerCalls == 0 && context.dispatchReturned == 0, Test,
462 "the abandoned timer stack returned through normal dispatch");
463 passed &= check(claimed == 0 && active == 0, Test,
464 "stack abandonment leaked a committed callback hazard");
466 context.unregisterSucceeded == 1 && context.unregisterReturned == 1 && reused && cleaned,
467 Test,
"cleanup did not release the synchronous unregister barrier");
469 NOTICE(
"HOSTED-WAIT-TEST: PASS timer-abandoned-dispatch-cleanup");
474bool timerDeferredSelfRemovalAbandonment() {
475 constexpr const char* Test =
"timer-abandoned-self-removal";
478 TimerAbandonContext context(timer, ®istry, TimerAbandonPoint::DeferredSelfRemoval);
480 &context,
nullptr,
false,
true,
true);
481 context.dispatcher->setName(
"hosted abandoned timer self-removal");
483 const bool registered = registry.registerHandler(&context.handler);
484 const bool started = registered && context.dispatcher->start();
485 const bool joined = started && context.dispatcher->join();
486 const size_t claimed = registry.claimedDispatchCountForTest();
487 const size_t active = registry.activeDispatchCountForTest(&context.handler);
488 const bool reused = registered && registry.registerHandler(&context.handler);
493 check(registered && started && joined, Test,
"the self-removing worker did not complete");
494 passed &= check(context.handlerCalls == 1 && context.selfRemovalRejected == 1 &&
495 context.dispatchReturned == 0 && context.failures == 0,
496 Test,
"the callback did not defer removal before abandoning its stack");
498 check(claimed == 0 && active == 0, Test,
"deferred self-removal leaked a callback hazard");
499 passed &= check(reused && cleaned, Test,
500 "final abandoned-stack cleanup did not retire and reuse the slot");
502 NOTICE(
"HOSTED-WAIT-TEST: PASS timer-abandoned-self-removal");
507struct HandlerLifetimeContext;
508HandlerLifetimeContext* g_HandlerLifetimeContext =
nullptr;
512 explicit LifetimeHandler(HandlerLifetimeContext& context) : m_Context(context) {}
514 void timer(uint64_t)
override;
517 HandlerLifetimeContext& m_Context;
520struct HandlerLifetimeContext {
521 explicit HandlerLifetimeContext(
Timer* timer)
527 hookObservedDrain(0),
529 callbacksAfterReturn(0),
530 unregisterReturned(0),
531 unregisterSucceeded(0),
532 selfRemovalPending(1),
533 selfRemovalRejected(0),
537 drainWaitPublished(0),
542 LifetimeHandler handler;
561void LifetimeHandler::timer(uint64_t) {
562 m_Context.handlerCalls += 1;
563 if (m_Context.unregisterReturned) {
564 m_Context.callbacksAfterReturn += 1;
566 if (m_Context.selfRemovalPending.compareAndSwap(1, 0) &&
567 !m_Context.timer->unregisterHandler(
this)) {
568 m_Context.selfRemovalRejected += 1;
569 if (!m_Context.timer->registerHandler(
this)) {
570 m_Context.revivalBlocked += 1;
577 explicit SelfRemovingHandler(
Timer* timer) : m_Timer(timer), calls(0), rejectionSeen(0) {}
579 void timer(uint64_t)
override {
581 if (!m_Timer->unregisterHandler(
this)) {
593 constexpr size_t YieldLimit = 10000;
594 HandlerLifetimeContext* context = g_HandlerLifetimeContext;
595 if (!context || thread != context->remover) {
599 context->waitHookCalls += 1;
601 if (!queue || !channel.owner || !channel.value || debugState != Thread::CallbackDrain ||
603 wait.channelOwner != channel.owner || wait.channelValue != channel.value || !wait.queued ||
604 wait.reason != WaitQueue::WakeReason::Waiting ||
605 !hasTimerCallbackDrainWait(thread, &context->handler)) {
606 context->waitHookFailures += 1;
610 context->drainWaitPublished += 1;
611 const uint64_t deadline =
612 context->timer->getTickCountNano() + (500 * Time::Multiplier::Millisecond);
613 for (
size_t i = 0; context->timer->getTickCountNano() < deadline && i < YieldLimit; ++i) {
616 currentWait.channelOwner == channel.owner && currentWait.channelValue == channel.value &&
617 currentWait.reason == WaitQueue::WakeReason::Signalled) {
618 context->wakeBeforeBlock += 1;
624 context->waitHookFailures += 1;
628 constexpr size_t YieldLimit = 10000;
629 HandlerLifetimeContext* context = g_HandlerLifetimeContext;
630 if (!context || handler != &context->handler || !context->phase.compareAndSwap(0, 1)) {
634 context->hookCalls += 1;
635 const uint64_t deadline =
636 context->timer->getTickCountNano() + (500 * Time::Multiplier::Millisecond);
637 bool observedDrain =
false;
638 for (
size_t i = 0; context->timer->getTickCountNano() < deadline && i < YieldLimit; ++i) {
639 if (context->phase ==
static_cast<size_t>(2) && context->drainWaitPublished &&
640 !context->unregisterReturned &&
641 hasTimerCallbackDrainWait(context->remover, &context->handler)) {
642 observedDrain =
true;
651 context->hookObservedDrain += 1;
653 context->failures += 1;
658int unregisterPinnedHandler(
void* parameter) {
659 HandlerLifetimeContext* context =
reinterpret_cast<HandlerLifetimeContext*
>(parameter);
660 const uint64_t deadline =
661 context->timer->getTickCountNano() + (500 * Time::Multiplier::Millisecond);
662 while (context->phase !=
static_cast<size_t>(1) &&
663 context->timer->getTickCountNano() < deadline) {
667 if (context->phase !=
static_cast<size_t>(1)) {
668 context->failures += 1;
673 if (context->timer->unregisterHandler(&context->handler)) {
674 context->unregisterSucceeded += 1;
676 context->unregisterReturned += 1;
681void ignoredTimerEvent(
size_t) {}
683void dispositionAHandler(
size_t) {
684 g_DispositionACalls += 1;
687void dispositionBHandler(
size_t) {
688 g_DispositionBCalls += 1;
691void relayDisposition(
Thread* thread) {
692 g_RelayCallbacks += 1;
693 Event*
event = g_RelayDisposition == 1 ? g_DispositionAEvent : g_DispositionBEvent;
699class TimerTestEvent :
public Event {
701 TimerTestEvent(
size_t number,
bool deletable,
702 uintptr_t handler =
reinterpret_cast<uintptr_t
>(&ignoredTimerEvent))
703 :
Event(handler, deletable), m_Number(number) {}
705 ~TimerTestEvent()
override {
706 if (m_Number == CullTestEventNumber) {
707 g_CullEventDestructions += 1;
723struct AlarmLinearizationContext;
725class AlarmLinearizationEvent :
public TimerTestEvent {
727 explicit AlarmLinearizationEvent(AlarmLinearizationContext& context);
728 ~AlarmLinearizationEvent()
override;
731 AlarmLinearizationContext& m_Context;
734struct AlarmLinearizationContext {
735 AlarmLinearizationContext(
Timer* timer,
Thread* owner)
739 removerReady(0, false),
742 lockBoundaryObserved(0),
749 AlarmLinearizationEvent* event;
759AlarmLinearizationContext* g_AlarmLinearizationContext =
nullptr;
761AlarmLinearizationEvent::AlarmLinearizationEvent(AlarmLinearizationContext& context)
762 : TimerTestEvent(AlarmLinearizationEventNumber, false), m_Context(context) {}
764AlarmLinearizationEvent::~AlarmLinearizationEvent() {
765 m_Context.destructions += 1;
768void observeAlarmSendAdmission(
Event* event) {
769 AlarmLinearizationContext* context = g_AlarmLinearizationContext;
770 if (!context || event != context->event) {
774 context->hookCalls += 1;
775 const bool lockHeld = HostedTimer::alarmLockHeldForTest();
777 if (lockHeld && interruptsDisabled && !context->removerReturned && !context->destructions) {
778 context->lockBoundaryObserved += 1;
780 context->failures += 1;
787 const uint64_t deadline =
788 context->timer->getTickCountNano() + (500 * Time::Multiplier::Millisecond);
789 while (!context->destructions && context->timer->getTickCountNano() < deadline) {
795int removeLinearizingAlarm(
void* parameter) {
796 AlarmLinearizationContext* context =
reinterpret_cast<AlarmLinearizationContext*
>(parameter);
797 context->removerReady.release();
799 const uint64_t deadline =
800 context->timer->getTickCountNano() + (500 * Time::Multiplier::Millisecond);
801 while (!context->phase && context->timer->getTickCountNano() < deadline) {
804 if (!context->phase) {
805 context->failures += 1;
809 context->timer->removeAlarm(context->event);
810 context->owner->cullEvent(context->event);
811 delete context->event;
812 context->event =
nullptr;
813 context->removerReturned = 1;
817bool check(
bool condition,
const char* test,
const char* detail) {
822 ERROR(
"HOSTED-WAIT-TEST: FAIL " << test <<
": " << detail);
826bool timerWriterLockIndependentDispatch() {
827 constexpr const char* Test =
"timer-dispatch-writer-lock-independent";
829 RegistryDispatchContext context(timer);
830 const bool registered = timer->registerHandler(&context.handler);
833 g_TimerRegistryDispatchContext = &context;
834 context.mutationRequested = 1;
835 const uint64_t deadline = timer->
getTickCountNano() + (250 * Time::Multiplier::Millisecond);
836 while (context.mutationRequested !=
static_cast<size_t>(3) &&
840 g_TimerRegistryDispatchContext =
nullptr;
843 const bool cleaned = registered && timer->unregisterHandler(&context.handler);
845 passed &= check(registered, Test,
"the test handler could not be registered");
846 passed &= check(context.mutationRequested == 3 && context.hookCalls == 1 &&
847 context.admitted == 1 && context.calls >= 2,
848 Test,
"dispatch or callback completion waited for the writer lock");
849 passed &= check(cleaned, Test,
"the test handler could not be removed");
852 "HOSTED-WAIT-TEST: PASS "
853 "timer-dispatch-writer-lock-independent");
859 RegistryDispatchContext* context = g_TimerRegistryDispatchContext;
860 if (!context || handler != &context->handler || !context->hookCalls.compareAndSwap(0, 1)) {
864 if (context->timer->unregisterHandler(&context->handler)) {
865 context->unregisterSucceeded += 1;
867 context->hookCalls = 2;
870bool timerPrePinUnregisterRevalidation() {
871 constexpr const char* Test =
"timer-pre-pin-unregister-revalidation";
873 RegistryDispatchContext context(timer);
875 g_TimerRegistryDispatchContext = &context;
876 HostedTimer::setHandlerPrePinHook(unregisterTimerBeforePin);
877 const bool registered = timer->registerHandler(&context.handler);
879 const uint64_t deadline = timer->
getTickCountNano() + (250 * Time::Multiplier::Millisecond);
880 while (context.hookCalls !=
static_cast<size_t>(2) && timer->
getTickCountNano() < deadline) {
884 HostedTimer::setHandlerPrePinHook(
nullptr);
885 g_TimerRegistryDispatchContext =
nullptr;
887 const bool reused = context.unregisterSucceeded && timer->registerHandler(&context.handler);
890 cleaned = timer->unregisterHandler(&context.handler);
891 }
else if (registered && !context.unregisterSucceeded) {
892 cleaned = timer->unregisterHandler(&context.handler);
896 passed &= check(registered, Test,
"the test handler could not be registered");
897 passed &= check(context.hookCalls == 2 && context.unregisterSucceeded == 1, Test,
898 "unregister did not retire the pre-pin publication");
900 check(context.calls == 0, Test,
"a stale pre-pin snapshot entered the retired callback");
901 passed &= check(reused && cleaned, Test,
"the revalidated slot could not be reused and removed");
904 "HOSTED-WAIT-TEST: PASS "
905 "timer-pre-pin-unregister-revalidation");
910bool timerAtomicDrainSelfRevival() {
911 constexpr const char* Test =
"timer-atomic-drain-self-revival";
913 AtomicDrainRaceContext context(timer);
915 g_AtomicDrainRaceContext = &context;
916 HostedTimer::setHandlerPinHook(atomicDrainPinHook);
917 HostedTimer::setHandlerAtomicDrainHook(atomicDrainTransitionHook);
919 &context,
nullptr,
false,
true);
920 remover->setName(
"hosted atomic timer-handler remover");
922 const bool removerWasReady = context.removerReady.acquireForCompletion(1, 2);
923 const bool registered = removerWasReady && timer->registerHandler(&context.handler);
926 context.beginDrain.release();
928 const bool joined = remover->
join();
929 HostedTimer::setHandlerAtomicDrainHook(
nullptr);
930 HostedTimer::setHandlerPinHook(
nullptr);
931 g_AtomicDrainRaceContext =
nullptr;
933 const size_t callsAtAtomicReturn = context.handlerCalls;
934 const uint64_t liveDeadline = timer->
getTickCountNano() + (250 * Time::Multiplier::Millisecond);
935 while (context.handlerCalls == callsAtAtomicReturn && timer->
getTickCountNano() < liveDeadline) {
938 const bool remainedLive = context.handlerCalls > callsAtAtomicReturn;
939 const bool cleaned = registered && timer->unregisterHandler(&context.handler);
943 check(removerWasReady, Test,
"the atomic remover did not become ready before registration");
944 passed &= check(registered, Test,
"the test handler could not be registered");
945 passed &= check(context.beginDrainTimedOut == 0, Test,
946 "the atomic remover timed out waiting for the pin hook");
947 passed &= check(context.pinTransitionTimedOut == 0, Test,
948 "the pin hook timed out waiting for the atomic drain transition");
949 passed &= check(context.drainCompletionTimedOut == 0, Test,
950 "the atomic drain hook timed out waiting for callback revival");
952 joined && context.failures == 0 && context.pinHookCalls == 1 && context.drainHookCalls == 1,
953 Test,
"the atomic drainer interleaving did not complete cleanly");
954 passed &= check(context.selfRemovalRejected == 1 && context.atomicRemovalRejected == 1 &&
955 context.revivalSucceeded == 1,
956 Test,
"the self-removal request was lost or could not be revived");
957 passed &= check(remainedLive && cleaned, Test,
958 "the revived handler was not live or could not be removed");
961 "HOSTED-WAIT-TEST: PASS "
962 "timer-atomic-drain-self-revival");
969 explicit ThreadContextTimerHandler(
Thread* registeringThread)
970 : registeringThread(registeringThread),
976 void timer(uint64_t delta)
override {
978 const size_t currentAddress =
reinterpret_cast<size_t>(current);
979 size_t expectedWorker = workerAddress.value();
980 if (!expectedWorker && current) {
981 workerAddress.compareAndSwap(0, currentAddress);
982 expectedWorker = workerAddress.value();
986 failed |= !current ? 1 : 0;
987 failed |= current == registeringThread ? 2 : 0;
988 failed |= expectedWorker != currentAddress ? 4 : 0;
990 failed |= current && current->getHostedSignalDepth() ? 16 : 0;
991 failed |= !delta || (delta % Time::Multiplier::Millisecond) ? 32 : 0;
994 failureMask |= failed;
999 Thread* registeringThread;
1006bool hostedTimerThreadContext(
Thread* registeringThread) {
1007 constexpr const char* Test =
"hosted-timer-thread-context";
1009 ThreadContextTimerHandler handler(registeringThread);
1010 const bool registered = timer->registerHandler(&handler);
1012 const uint64_t deadline = timer->
getTickCountNano() + (250 * Time::Multiplier::Millisecond);
1013 while (registered && handler.calls <
static_cast<size_t>(2) &&
1018 const bool removed = registered && timer->unregisterHandler(&handler);
1020 check(registered && removed && handler.calls >=
static_cast<size_t>(2) &&
1021 handler.failures == 0 && handler.workerAddress !=
static_cast<size_t>(0),
1022 Test,
"timer callbacks did not remain on one ordinary worker context");
1024 ERROR(
"HOSTED-WAIT-TEST: INFO "
1025 << Test <<
": calls=" << handler.calls.value()
1026 <<
", failures=" << handler.failures.value() <<
", mask=" <<
Hex
1027 << handler.failureMask.value() <<
", worker=" << handler.workerAddress.value()
1028 <<
", registrar=" <<
reinterpret_cast<uintptr_t
>(registeringThread));
1031 NOTICE(
"HOSTED-WAIT-TEST: PASS hosted-timer-thread-context");
1038 HostedTimerOverrunHandler() : calls(0), maximumDelta(0) {}
1040 void timer(uint64_t delta)
override {
1041 uint64_t maximum = maximumDelta.value();
1042 while (delta > maximum && !maximumDelta.compareAndSwap(maximum, delta)) {
1043 maximum = maximumDelta.value();
1052bool hostedTimerOverrunAccounting() {
1053 constexpr const char* Test =
"hosted-timer-overrun-accounting";
1055 HostedTimerOverrunHandler handler;
1056 const bool registered = timer->registerHandler(&handler);
1058 const uint64_t baselineDeadline =
1060 while (registered && !handler.calls && timer->
getTickCountNano() < baselineDeadline) {
1064 const bool baselineObserved = handler.calls !=
static_cast<size_t>(0);
1065 handler.maximumDelta = 0;
1067 if (baselineObserved && interruptsWereEnabled) {
1069 const uint64_t blockedUntil = timer->
getTickCountNano() + (30 * Time::Multiplier::Millisecond);
1076 const uint64_t overrunDeadline =
1078 while (handler.maximumDelta <= Time::Multiplier::Millisecond &&
1083 const bool removed = registered && timer->unregisterHandler(&handler);
1084 const uint64_t maximumDelta = handler.maximumDelta.value();
1086 check(registered && baselineObserved && interruptsWereEnabled && removed &&
1087 maximumDelta > Time::Multiplier::Millisecond &&
1088 !(maximumDelta % Time::Multiplier::Millisecond),
1089 Test,
"deferred POSIX expirations were not preserved in callback delta");
1091 NOTICE(
"HOSTED-WAIT-TEST: PASS hosted-timer-overrun-accounting");
1096bool timerClockAndDeadline(
Thread* thread) {
1103 passed &= check(millis >= nanosBefore / Time::Multiplier::Millisecond &&
1104 millis <= nanosAfter / Time::Multiplier::Millisecond,
1105 "timer-clock-deadline",
1106 "millisecond and nanosecond tick APIs use different clocks or units");
1108 TimerTestEvent* remainingEvent =
new TimerTestEvent(TimerTestEventNumber,
false);
1109 timer->
addAlarm(remainingEvent, 2);
1110 const size_t remaining = timer->
removeAlarm(remainingEvent,
false);
1111 delete remainingEvent;
1112 passed &= check(remaining == 2,
"timer-clock-deadline",
1113 "alarm removal did not return whole remaining seconds");
1116 TimerTestEvent* overflowEvent =
new TimerTestEvent(TimerTestEventNumber,
false);
1118 timer->
addAlarm(overflowEvent, ~
static_cast<size_t>(0), ~
static_cast<size_t>(0));
1119 while (timer->
getTickCountNano() < overflowStart + (5 * Time::Multiplier::Millisecond)) {
1122 const bool overflowFired = thread->
hasEvent(overflowEvent);
1125 delete overflowEvent;
1126 passed &= check(!overflowFired,
"timer-clock-deadline",
1127 "an overflowing alarm duration wrapped into an immediate deadline");
1129 TimerTestEvent* deadlineEvent =
new TimerTestEvent(TimerTestEventNumber,
false);
1131 timer->
addAlarm(deadlineEvent, 0, 20000);
1133 const uint64_t earlyDeadline = start + (5 * Time::Multiplier::Millisecond);
1137 const bool firedEarly = thread->
hasEvent(deadlineEvent);
1139 const uint64_t lateDeadline = start + (250 * Time::Multiplier::Millisecond);
1143 const bool fired = thread->
hasEvent(deadlineEvent);
1147 const bool removed = !thread->
hasEvent(deadlineEvent);
1148 delete deadlineEvent;
1151 passed &= check(!firedEarly,
"timer-clock-deadline",
"a 20ms alarm fired during its first 5ms");
1152 passed &= check(fired,
"timer-clock-deadline",
"a 20ms alarm did not publish within 250ms");
1153 passed &= check(removed,
"timer-clock-deadline",
"a cancelled alarm event remained queued");
1156 NOTICE(
"HOSTED-WAIT-TEST: PASS timer-clock-deadline");
1161bool timerAlarmSendLinearization() {
1162 constexpr const char* Test =
"timer-alarm-send-linearization";
1165 AlarmLinearizationContext context(timer, owner);
1166 context.event =
new AlarmLinearizationEvent(context);
1167 owner->inhibitEvent(AlarmLinearizationEventNumber,
true);
1171 g_AlarmLinearizationContext = &context;
1172 HostedTimer::setAlarmSendAdmissionHookForTest(observeAlarmSendAdmission);
1176 &context,
nullptr,
false,
true);
1177 remover->setName(
"hosted alarm linearization remover");
1178 const bool removerReady = context.removerReady.acquireForCompletion(1, 2);
1180 timer->
addAlarm(context.event, 0, 1000);
1182 const bool removerJoined = remover->
join();
1185 HostedTimer::setAlarmSendAdmissionHookForTest(
nullptr);
1186 g_AlarmLinearizationContext =
nullptr;
1189 if (context.event) {
1191 owner->cullEvent(context.event);
1192 delete context.event;
1193 context.event =
nullptr;
1195 owner->inhibitEvent(AlarmLinearizationEventNumber,
false);
1198 check(interruptsWereEnabled && removerReady && removerJoined && context.hookCalls == 1 &&
1199 context.lockBoundaryObserved == 1 && context.removerReturned == 1 &&
1200 context.destructions == 1 && context.failures == 0,
1201 Test,
"alarm cancellation crossed the Event send-admission boundary");
1203 NOTICE(
"HOSTED-WAIT-TEST: PASS timer-alarm-send-linearization");
1208bool timerAlarmRemovalLifetime() {
1209 constexpr size_t AlarmBatchSize = 17;
1210 constexpr size_t AlarmDelaySeconds = ~static_cast<size_t>(0);
1212 TimerTestEvent* events[AlarmBatchSize] = {};
1217 for (
size_t i = 0; i < AlarmBatchSize; ++i) {
1218 events[i] =
new TimerTestEvent(TimerTestEventNumber,
false);
1219 timer->
addAlarm(events[i], AlarmDelaySeconds);
1221 for (
size_t i = 0; i < AlarmBatchSize; ++i) {
1223 passed &= timer->
removeAlarm(events[i],
false) == 0;
1227 for (
size_t i = 0; i < AlarmBatchSize; ++i) {
1228 events[i] =
new TimerTestEvent(TimerTestEventNumber,
false);
1229 timer->
addAlarm(events[i], AlarmDelaySeconds);
1231 for (
size_t i = 0; i < AlarmBatchSize; ++i) {
1232 passed &= timer->
removeAlarm(events[i],
false) > 0;
1233 passed &= timer->
removeAlarm(events[i],
false) == 0;
1237 passed &= check(passed,
"timer-alarm-removal-lifetime",
1238 "an alarm removal overload left its alarm registered");
1240 NOTICE(
"HOSTED-WAIT-TEST: PASS timer-alarm-removal-lifetime");
1245bool timerHandlerLifetimeBarrier() {
1247 HandlerLifetimeContext context(timer);
1249 &context,
nullptr,
false,
true);
1250 context.remover = remover;
1251 remover->setName(
"hosted timer-handler remover");
1253 g_HandlerLifetimeContext = &context;
1254 HostedTimer::setHandlerPinHook(handlerPinHook);
1255 WaitQueue::setBeforeBlockHook(timerDrainBeforeBlockHook);
1256 const bool registered = timer->registerHandler(&context.handler);
1257 const bool joined = remover->
join();
1258 WaitQueue::setBeforeBlockHook(
nullptr);
1259 HostedTimer::setHandlerPinHook(
nullptr);
1260 g_HandlerLifetimeContext =
nullptr;
1262 if (!context.unregisterSucceeded) {
1263 timer->unregisterHandler(&context.handler);
1266 const size_t callsAtUnregisterReturn = context.handlerCalls;
1267 const uint64_t postReturnDeadline =
1273 SelfRemovingHandler selfRemoving(timer);
1274 const bool selfRegistered = timer->registerHandler(&selfRemoving);
1275 const uint64_t selfDeadline = timer->
getTickCountNano() + (250 * Time::Multiplier::Millisecond);
1279 const size_t selfCallsAfterRetirement = selfRemoving.calls;
1280 const uint64_t selfQuietDeadline =
1288 const bool selfReregistered = timer->registerHandler(&selfRemoving);
1289 const bool selfCleanup = selfReregistered && timer->unregisterHandler(&selfRemoving);
1292 passed &= check(registered,
"timer-handler-lifetime",
"the test handler could not be registered");
1293 passed &= check(joined && context.failures == 0,
"timer-handler-lifetime",
1294 "the concurrent unregister worker did not complete cleanly");
1296 check(context.hookCalls == 1 && context.hookObservedDrain == 1,
"timer-handler-lifetime",
1297 "unregister returned instead of waiting for the pinned callback");
1298 const bool waitQueueDrainPassed =
1299 registered && joined && context.waitHookCalls == 1 && context.waitHookFailures == 0 &&
1300 context.drainWaitPublished == 1 && context.wakeBeforeBlock == 1 &&
1301 context.hookObservedDrain == 1 && context.unregisterSucceeded == 1 &&
1302 context.unregisterReturned == 1;
1303 passed &= check(waitQueueDrainPassed,
"timer-handler-waitqueue-drain",
1304 "callback drain did not publish and consume a pre-block wake");
1305 if (waitQueueDrainPassed) {
1306 NOTICE(
"HOSTED-WAIT-TEST: PASS timer-handler-waitqueue-drain");
1308 passed &= check(context.unregisterSucceeded == 1 && context.unregisterReturned == 1,
1309 "timer-handler-lifetime",
"the pinned handler did not unregister successfully");
1310 const bool selfDrainRacePassed =
1311 registered && joined && context.failures == 0 && context.hookObservedDrain == 1 &&
1312 context.selfRemovalPending == 0 && context.selfRemovalRejected == 1 &&
1313 context.unregisterSucceeded == 1 && context.unregisterReturned == 1 &&
1314 context.revivalBlocked == 1;
1315 passed &= check(selfDrainRacePassed,
"timer-self-unregister-vs-drainer",
1316 "self-removal escaped or revived a synchronous drain");
1317 if (selfDrainRacePassed) {
1319 "HOSTED-WAIT-TEST: PASS "
1320 "timer-self-unregister-vs-drainer");
1323 check(context.handlerCalls >= 1 && context.handlerCalls == callsAtUnregisterReturn &&
1324 context.callbacksAfterReturn == 0,
1325 "timer-handler-lifetime",
"a timer callback began after unregisterHandler returned");
1327 check(selfRegistered && selfRemoving.rejectionSeen == 1 && selfCallsAfterRetirement == 1 &&
1328 selfRemoving.calls == selfCallsAfterRetirement,
1329 "timer-handler-lifetime",
1330 "self-unregister was not rejected and retired after callback return");
1331 passed &= check(selfReregistered && selfCleanup,
"timer-handler-lifetime",
1332 "deferred self-removal did not release its registry slot");
1335 NOTICE(
"HOSTED-WAIT-TEST: PASS timer-handler-lifetime");
1340struct SemaphoreTimeoutContext {
1342 : semaphore(semaphore),
waiter(
waiter), sawTimeoutEvent(0), failures(0) {}
1350int releaseAfterTimeoutQueued(
void* parameter) {
1351 SemaphoreTimeoutContext* context =
reinterpret_cast<SemaphoreTimeoutContext*
>(parameter);
1353 const uint64_t deadline = timer->
getTickCountNano() + (500 * Time::Multiplier::Millisecond);
1355 while (!context->waiter->hasEvent(EventNumbers::Interrupt) &&
1360 if (!context->waiter->hasEvent(EventNumbers::Interrupt)) {
1361 context->failures += 1;
1365 context->sawTimeoutEvent += 1;
1366 context->semaphore->release();
1370bool semaphoreQueuedTimeoutCancellation(
Thread* thread) {
1372 SemaphoreTimeoutContext context(&semaphore, thread);
1374 &context,
nullptr,
false,
true);
1375 releaser->setName(
"hosted queued-timeout releaser");
1378 Semaphore::SemaphoreError error = Semaphore::NoError;
1379 const bool acquired = semaphore.acquireWithError(1, 0, 20000, error);
1380 const bool joined = releaser->
join();
1381 const bool eventRemoved = !thread->
hasEvent(EventNumbers::Interrupt);
1385 passed &= check(acquired && error == Semaphore::NoError,
"semaphore-timeout-cancel",
1386 "a signalled waiter reported the queued timeout instead");
1388 check(joined && context.failures == 0 && context.sawTimeoutEvent == 1,
1389 "semaphore-timeout-cancel",
"the releaser did not observe exactly one queued timeout");
1391 check(eventRemoved,
"semaphore-timeout-cancel",
"the losing timeout event remained queued");
1392 passed &= check(semaphore.getValue() == 0,
"semaphore-timeout-cancel",
1393 "the released semaphore unit was not consumed");
1396 NOTICE(
"HOSTED-WAIT-TEST: PASS fresh-thread-timer-progress");
1397 NOTICE(
"HOSTED-WAIT-TEST: PASS semaphore-timeout-cancel");
1402bool relayUsesLatestDisposition(
Thread* thread) {
1404 TimerTestEvent dispositionA(DispositionAEventNumber,
false,
1405 reinterpret_cast<uintptr_t
>(&dispositionAHandler));
1406 TimerTestEvent dispositionB(DispositionBEventNumber,
false,
1407 reinterpret_cast<uintptr_t
>(&dispositionBHandler));
1408 RelayEvent relay(&relayDisposition, RelayTestEventNumber);
1410 g_RelayCallbacks = 0;
1411 g_DispositionACalls = 0;
1412 g_DispositionBCalls = 0;
1413 g_DispositionAEvent = &dispositionA;
1414 g_DispositionBEvent = &dispositionB;
1416 g_RelayDisposition = 1;
1418 g_RelayDisposition = 2;
1420 const uint64_t deadline = timer->
getTickCountNano() + (250 * Time::Multiplier::Millisecond);
1429 g_DispositionAEvent =
nullptr;
1430 g_DispositionBEvent =
nullptr;
1433 check(g_RelayCallbacks == 1 && g_DispositionACalls == 0 && g_DispositionBCalls == 1,
1434 "relay-latest-disposition",
"timer expiry retained the disposition from arm time");
1436 NOTICE(
"HOSTED-WAIT-TEST: PASS relay-latest-disposition");
1441bool exactCullRetainsOwnership(
Thread* thread) {
1442 const size_t destructionsBefore = g_CullEventDestructions;
1443 TimerTestEvent*
event =
new TimerTestEvent(CullTestEventNumber,
true);
1446 const bool queued = thread->
sendEvent(event);
1448 const bool removed = !thread->
hasEvent(event);
1449 const bool retained = g_CullEventDestructions == destructionsBefore;
1457 passed &= check(queued && removed,
"timeoutguard-cancel-ownership",
1458 "the exact queued event was not culled");
1459 passed &= check(retained,
"timeoutguard-cancel-ownership",
1460 "exact-event culling deleted caller-owned storage");
1462 check(g_CullEventDestructions == destructionsBefore + 1,
"timeoutguard-cancel-ownership",
1463 "caller-owned event destruction did not occur exactly once");
1466 NOTICE(
"HOSTED-WAIT-TEST: PASS timeoutguard-cancel-ownership");
1472bool runHostedTimerRegressions(
Thread* thread) {
1474 hostedTimerOverrunAccounting() && timerWriterLockIndependentDispatch() &&
1475 timerPrePinUnregisterRevalidation() && timerAtomicDrainSelfRevival() &&
1476 timerPartialHazardAbandonment() && timerCommittedHazardAbandonment() &&
1477 timerDeferredSelfRemovalAbandonment() && timerClockAndDeadline(thread) &&
1478 timerAlarmSendLinearization() && timerAlarmRemovalLifetime() &&
1479 timerHandlerLifetimeBarrier() && semaphoreQueuedTimeoutCancellation(thread) &&
1480 relayUsesLatestDisposition(thread) && exactCullRetainsOwnership(thread);
virtual size_t getNumber()=0
virtual size_t serialize(uint8_t *pBuffer)=0
virtual Timer * getTimer()=0
static bool getInterrupts()
static ProcessorInformation & information()
static void setInterrupts(bool bEnable)
static Scheduler & instance()
bool getWaitDebugInfo(WaitDebugInfo &info)
bool hasEvent(Event *pEvent)
void cullEvent(Event *pEvent)
DebugState getDebugState(uintptr_t &address)
bool sendEvent(Event *pEvent)
void inhibitEvent(size_t eventNumber, bool bInhibit)
bool unregisterHandler(TimerHandler *handler)
virtual void timer(uint64_t delta)=0
virtual uint64_t getTickCount()=0
virtual void removeAlarm(class Event *pEvent)=0
virtual uint64_t getTickCountNano()
virtual void addAlarm(class Event *pEvent, size_t alarmSecs, size_t alarmUsecs=0)=0
const size_t UserStart
Start of user-defined events.