8#include "pedigree/kernel/Atomic.h"
9#include "pedigree/kernel/Log.h"
10#include "pedigree/kernel/process/Scheduler.h"
11#include "pedigree/kernel/process/Semaphore.h"
12#include "pedigree/kernel/process/Thread.h"
14#include "modules/drivers/common/usb-hcd/CallbackDelivery.h"
15#include "modules/drivers/common/usb-hcd/TransferCompletion.h"
16#include "modules/system/usb/Usb.h"
21bool check(
bool condition,
const char* test,
const char* detail) {
25 ERROR(
"HOSTED-WAIT-TEST: FAIL " << test <<
": " << detail);
29void countDestruction(
void* parameter) {
34struct PendingStealContext {
35 PendingStealContext(DeliveryQueue* queue,
const DeliveryQueue::Key& key)
49 DeliveryQueue::Key pendingKey;
61void stolenCallback(uintptr_t parameter, ssize_t result) {
62 auto* context =
reinterpret_cast<PendingStealContext*
>(parameter);
63 context->stolenCalls += 1;
64 context->stolenOrder = context->sequence += 1;
66 context->failures += 1;
69void releaseAfterStolenCallback(
void* parameter) {
70 auto* context =
reinterpret_cast<PendingStealContext*
>(parameter);
71 context->releaseOrder = context->sequence += 1;
74void firstCallback(uintptr_t parameter, ssize_t result) {
75 auto* context =
reinterpret_cast<PendingStealContext*
>(parameter);
76 context->firstCalls += 1;
77 context->firstOrder = context->sequence += 1;
79 context->failures += 1;
80 if (context->queue->drain(context->pendingKey))
81 context->drainSucceeded += 1;
82 context->resumedOrder = context->sequence += 1;
85bool pendingRecordCanBeStolen() {
88 const DeliveryQueue::Key firstKey = {0x100, queue.nextGeneration()};
89 const DeliveryQueue::Key pendingKey = {0x101, queue.nextGeneration()};
90 PendingStealContext context(&queue, pendingKey);
92 DeliveryQueue::Record* first =
93 queue.create(firstKey, firstCallback,
reinterpret_cast<uintptr_t
>(&context), 11,
nullptr,
94 nullptr, countDestruction, &destroyed);
95 DeliveryQueue::Record* pending =
96 queue.create(pendingKey, stolenCallback,
reinterpret_cast<uintptr_t
>(&context), 22,
97 releaseAfterStolenCallback, &context, countDestruction, &destroyed);
101 queue.publish(records);
103 queue.deliver(first);
104 queue.deliver(pending);
106 const bool passed = check(
107 context.firstCalls == 1 && context.stolenCalls == 1 && context.drainSucceeded == 1 &&
108 context.failures == 0 && context.firstOrder == 1 && context.stolenOrder == 2 &&
109 context.releaseOrder == 3 && context.resumedOrder == 4 && destroyed == 2 && queue.empty(),
110 "usb-callback-pending-steal",
111 "a callback could not synchronously steal a later captured callback");
113 NOTICE(
"HOSTED-WAIT-TEST: PASS usb-callback-pending-steal");
117struct SelfDrainContext {
118 SelfDrainContext(DeliveryQueue* queue,
const DeliveryQueue::Key& key)
119 : queue(queue), key(key), calls(0), drained(0) {}
121 DeliveryQueue* queue;
122 DeliveryQueue::Key key;
127void selfDrainingCallback(uintptr_t parameter, ssize_t) {
128 auto* context =
reinterpret_cast<SelfDrainContext*
>(parameter);
130 if (context->queue->drain(context->key))
131 context->drained += 1;
134bool runningRecordCanDrainItself() {
137 const DeliveryQueue::Key key = {0x200, queue.nextGeneration()};
138 SelfDrainContext context(&queue, key);
139 DeliveryQueue::Record* record =
140 queue.create(key, selfDrainingCallback,
reinterpret_cast<uintptr_t
>(&context), 0,
nullptr,
141 nullptr, countDestruction, &destroyed);
144 queue.publish(records);
145 queue.deliver(record);
147 const bool passed = check(
148 context.calls == 1 && context.drained == 1 && destroyed == 1 && queue.empty(),
149 "usb-callback-self-drain",
"a callback draining its own generation blocked or ran twice");
151 NOTICE(
"HOSTED-WAIT-TEST: PASS usb-callback-self-drain");
155struct RunningDrainContext {
156 RunningDrainContext(DeliveryQueue* queue,
const DeliveryQueue::Key& key)
161 allowCallbackReturn(0),
168 DeliveryQueue* queue;
169 DeliveryQueue::Key key;
170 DeliveryQueue::Record* record;
180void blockingCallback(uintptr_t parameter, ssize_t) {
181 auto* context =
reinterpret_cast<RunningDrainContext*
>(parameter);
182 context->callbackCalls += 1;
183 context->callbackEntered.release();
184 const bool released = context->allowCallbackReturn.acquireForCompletion();
188int deliverRunningRecord(
void* parameter) {
189 auto* context =
reinterpret_cast<RunningDrainContext*
>(parameter);
190 context->queue->deliver(context->record);
191 context->deliveryReturned += 1;
195int drainRunningRecord(
void* parameter) {
196 auto* context =
reinterpret_cast<RunningDrainContext*
>(parameter);
197 context->drainEntered.release();
198 if (context->queue->drain(context->key))
199 context->drainSucceeded += 1;
200 context->drainReturned += 1;
204bool waitUntilSleeping(
Thread* thread) {
205 for (
size_t attempt = 0; attempt < 10000; ++attempt) {
206 if (thread->
getStatus() == Thread::Sleeping)
213bool anotherThreadWaitsForRunningRecord() {
216 const DeliveryQueue::Key key = {0x300, queue.nextGeneration()};
217 RunningDrainContext context(&queue, key);
218 context.record = queue.create(key, blockingCallback,
reinterpret_cast<uintptr_t
>(&context), 0,
219 nullptr,
nullptr, countDestruction, &destroyed);
222 queue.publish(records);
226 new Thread(kernelProcess, deliverRunningRecord, &context,
nullptr,
false,
true);
227 delivery->setName(
"hosted USB callback delivery");
228 const bool callbackEntered = context.callbackEntered.acquireForCompletion();
230 Thread* drainer =
new Thread(kernelProcess, drainRunningRecord, &context,
nullptr,
false,
true);
231 drainer->setName(
"hosted USB callback drainer");
232 const bool drainEntered = context.drainEntered.acquireForCompletion();
233 const bool drainBlocked = waitUntilSleeping(drainer);
234 const bool returnedEarly =
static_cast<size_t>(context.drainReturned) != 0;
236 context.allowCallbackReturn.release();
237 const bool deliveryJoined = delivery->
join();
238 const bool drainerJoined = drainer->
join();
240 const bool passed = check(callbackEntered && drainEntered && drainBlocked && !returnedEarly &&
241 deliveryJoined && drainerJoined && context.callbackCalls == 1 &&
242 context.deliveryReturned == 1 && context.drainReturned == 1 &&
243 context.drainSucceeded == 1 && destroyed == 1 && queue.empty(),
244 "usb-callback-running-drain",
245 "a cross-thread drain did not wait for the running callback");
247 NOTICE(
"HOSTED-WAIT-TEST: PASS usb-callback-running-drain");
251bool producerWaitsForStolenRunningRecord() {
254 const DeliveryQueue::Key key = {0x350, queue.nextGeneration()};
255 RunningDrainContext context(&queue, key);
256 context.record = queue.create(key, blockingCallback,
reinterpret_cast<uintptr_t
>(&context), 0,
257 nullptr,
nullptr, countDestruction, &destroyed);
260 queue.publish(records);
263 Thread* drainer =
new Thread(kernelProcess, drainRunningRecord, &context,
nullptr,
false,
true);
264 drainer->setName(
"hosted USB callback stealer");
265 const bool drainEntered = context.drainEntered.acquireForCompletion();
266 const bool callbackEntered = context.callbackEntered.acquireForCompletion();
269 new Thread(kernelProcess, deliverRunningRecord, &context,
nullptr,
false,
true);
270 producer->setName(
"hosted USB callback producer");
271 const bool producerBlocked = waitUntilSleeping(producer);
272 const bool returnedEarly =
static_cast<size_t>(context.deliveryReturned) != 0;
274 context.allowCallbackReturn.release();
275 const bool drainerJoined = drainer->
join();
276 const bool producerJoined = producer->
join();
278 const bool passed = check(drainEntered && callbackEntered && producerBlocked && !returnedEarly &&
279 drainerJoined && producerJoined && context.callbackCalls == 1 &&
280 context.deliveryReturned == 1 && context.drainReturned == 1 &&
281 context.drainSucceeded == 1 && destroyed == 1 && queue.empty(),
282 "usb-callback-producer-drains-steal",
283 "the producer abandoned a callback stolen by another thread");
285 NOTICE(
"HOSTED-WAIT-TEST: PASS usb-callback-producer-drains-steal");
290 CountContext() : calls(0) {}
295void countCallback(uintptr_t parameter, ssize_t) {
296 auto* context =
reinterpret_cast<CountContext*
>(parameter);
300bool generationIsPartOfIdentity() {
303 CountContext context;
304 const size_t generation = queue.nextGeneration();
305 const DeliveryQueue::Key key = {0x400, generation};
306 const DeliveryQueue::Key wrongGeneration = {0x400, generation + 1};
307 DeliveryQueue::Record* record =
308 queue.create(key, countCallback,
reinterpret_cast<uintptr_t
>(&context), 0,
nullptr,
nullptr,
309 countDestruction, &destroyed);
312 queue.publish(records);
314 const bool wrongDrained = queue.drain(wrongGeneration);
315 queue.deliver(record);
317 const bool passed = check(!wrongDrained && context.calls == 1 && destroyed == 1 && queue.empty(),
318 "usb-callback-generation-identity",
319 "a stale transaction generation matched a captured callback");
321 NOTICE(
"HOSTED-WAIT-TEST: PASS usb-callback-generation-identity");
325bool allPendingRecordsCanBeDrained() {
328 CountContext context;
330 for (
size_t i = 0; i < 3; ++i) {
331 const DeliveryQueue::Key key = {0x500 + i, queue.nextGeneration()};
332 records.
pushBack(queue.create(key, countCallback,
reinterpret_cast<uintptr_t
>(&context), 0,
333 nullptr,
nullptr, countDestruction, &destroyed));
335 queue.publish(records);
337 const size_t drained = queue.drainAll();
338 const bool emptyAfterDrain = queue.empty();
339 while (records.
count())
342 const bool passed = check(
343 drained == 3 && emptyAfterDrain && context.calls == 3 && destroyed == 3 && queue.empty(),
344 "usb-callback-drain-all",
"controller teardown could not drain every published callback");
346 NOTICE(
"HOSTED-WAIT-TEST: PASS usb-callback-drain-all");
350bool recurringCancellationSuppressesPendingSamples() {
353 CountContext context;
354 constexpr uintptr_t Transaction = 0x600;
355 constexpr size_t Subscription = 0x61;
356 constexpr size_t OtherSubscription = 0x62;
358 DeliveryQueue::Record* first = queue.create({Transaction, queue.nextGeneration(), Subscription},
359 countCallback,
reinterpret_cast<uintptr_t
>(&context),
360 0,
nullptr,
nullptr, countDestruction, &destroyed);
361 DeliveryQueue::Record* second = queue.create({Transaction, queue.nextGeneration(), Subscription},
362 countCallback,
reinterpret_cast<uintptr_t
>(&context),
363 0,
nullptr,
nullptr, countDestruction, &destroyed);
364 DeliveryQueue::Record* other = queue.create(
365 {Transaction, queue.nextGeneration(), OtherSubscription}, countCallback,
366 reinterpret_cast<uintptr_t
>(&context), 0,
nullptr,
nullptr, countDestruction, &destroyed);
370 queue.publish(records);
372 const bool cancelled = queue.cancelSubscription(Transaction, Subscription);
373 const bool generationScoped = queue.activeCount() == 1;
374 queue.deliver(first);
375 queue.deliver(second);
376 queue.deliver(other);
379 check(cancelled && generationScoped && context.calls == 1 && destroyed == 3 && queue.empty(),
380 "usb-callback-recurring-cancel",
381 "recurring cancellation invoked a pending callback or consumed a reused generation");
383 NOTICE(
"HOSTED-WAIT-TEST: PASS usb-callback-recurring-cancel");
387struct ReciprocalSubscriptionContext {
388 ReciprocalSubscriptionContext(DeliveryQueue* first, DeliveryQueue* second)
392 cancellationsAttempted(0),
396 DeliveryQueue* first;
397 DeliveryQueue* second;
404void firstReciprocalSubscriptionCallback(uintptr_t parameter, ssize_t) {
405 auto* context =
reinterpret_cast<ReciprocalSubscriptionContext*
>(parameter);
406 context->callbacksEntered += 1;
407 while (context->callbacksEntered !=
static_cast<size_t>(2))
409 if (!context->second->cancelSubscription(0x701, 0x72))
410 context->resetRejected += 1;
412 context->failures += 1;
413 context->cancellationsAttempted += 1;
414 while (context->cancellationsAttempted !=
static_cast<size_t>(2))
418void secondReciprocalSubscriptionCallback(uintptr_t parameter, ssize_t) {
419 auto* context =
reinterpret_cast<ReciprocalSubscriptionContext*
>(parameter);
420 context->callbacksEntered += 1;
421 while (context->callbacksEntered !=
static_cast<size_t>(2))
423 if (!context->first->cancelSubscription(0x700, 0x71))
424 context->resetRejected += 1;
426 context->failures += 1;
427 context->cancellationsAttempted += 1;
428 while (context->cancellationsAttempted !=
static_cast<size_t>(2))
432struct ReciprocalDeliveryThread {
433 DeliveryQueue* queue;
434 DeliveryQueue::Record* record;
437int deliverReciprocalSubscription(
void* parameter) {
438 auto* delivery =
reinterpret_cast<ReciprocalDeliveryThread*
>(parameter);
439 delivery->queue->deliver(delivery->record);
443bool reciprocalSubscriptionCancellationDoesNotDeadlock() {
445 DeliveryQueue second;
446 ReciprocalSubscriptionContext context(&first, &second);
447 DeliveryQueue::Record* firstRecord =
448 first.create({0x700, first.nextGeneration(), 0x71}, firstReciprocalSubscriptionCallback,
449 reinterpret_cast<uintptr_t
>(&context), 0);
450 DeliveryQueue::Record* secondRecord =
451 second.create({0x701, second.nextGeneration(), 0x72}, secondReciprocalSubscriptionCallback,
452 reinterpret_cast<uintptr_t
>(&context), 0);
456 secondRecords.
pushBack(secondRecord);
457 first.publish(firstRecords);
458 second.publish(secondRecords);
460 ReciprocalDeliveryThread firstDelivery = {&first, firstRecord};
461 ReciprocalDeliveryThread secondDelivery = {&second, secondRecord};
464 new Thread(process, deliverReciprocalSubscription, &firstDelivery,
nullptr,
false,
true);
466 new Thread(process, deliverReciprocalSubscription, &secondDelivery,
nullptr,
false,
true);
467 firstThread->setName(
"hosted USB reciprocal callback one");
468 secondThread->setName(
"hosted USB reciprocal callback two");
469 const bool firstJoined = firstThread->
join();
470 const bool secondJoined = secondThread->
join();
472 const bool externallyRetired =
473 first.cancelSubscription(0x700, 0x71) && second.cancelSubscription(0x701, 0x72);
475 check(firstJoined && secondJoined && externallyRetired && context.callbacksEntered == 2 &&
476 context.cancellationsAttempted == 2 && context.resetRejected == 2 &&
477 context.failures == 0 && first.empty() && second.empty(),
478 "usb-callback-reciprocal-cancel",
479 "callbacks on separate queues waited on each other during cancellation");
481 NOTICE(
"HOSTED-WAIT-TEST: PASS usb-callback-reciprocal-cancel");
485bool capturedCompletionHasOnePublisher() {
489 CountContext context;
490 completion.
arm(countCallback,
reinterpret_cast<uintptr_t
>(&context), 41);
494 const bool claimedTwice = completion.
claimCaptured(duplicate);
495 const bool teardownClaimed = completion.
claimForTeardown(-TransactionError, duplicate);
497 const bool passed = check(
498 captured && claimed && !claimedTwice && !teardownClaimed && claim.callback == countCallback &&
499 claim.parameter ==
reinterpret_cast<uintptr_t
>(&context) && claim.generation == 41 &&
500 claim.result == 73 && claim.reason == UsbHcd::TransferCompletion::Reason::Natural &&
501 completion.state() == UsbHcd::TransferCompletion::State::PublicationClaimed,
502 "usb-completion-captured-exactly-once",
503 "a captured hardware result had more than one callback publisher");
506 "HOSTED-WAIT-TEST: PASS "
507 "usb-completion-captured-exactly-once");
511bool cancellationOwnsCompletion() {
514 CountContext context;
515 completion.
arm(countCallback,
reinterpret_cast<uintptr_t
>(&context), 42);
518 countCallback,
reinterpret_cast<uintptr_t
>(&context), -TransactionError, claim);
520 const bool ordinaryClaimed = completion.
claimCaptured(claim);
521 const bool teardownClaimed = completion.
claimForTeardown(-TransactionError, claim);
524 check(cancellation == UsbHcd::TransferCompletion::CancellationDisposition::Claimed &&
525 !capturedAfterCancel && !ordinaryClaimed && !teardownClaimed &&
526 claim.callback == countCallback &&
527 claim.parameter ==
reinterpret_cast<uintptr_t
>(&context) &&
528 claim.generation == 42 && claim.result == -TransactionError &&
529 claim.reason == UsbHcd::TransferCompletion::Reason::Cancelled &&
530 completion.state() == UsbHcd::TransferCompletion::State::PublicationClaimed,
531 "usb-completion-cancellation-ownership",
532 "completion ownership escaped a successful synchronous cancellation");
534 NOTICE(
"HOSTED-WAIT-TEST: PASS usb-completion-cancellation-ownership");
538bool teardownTerminalizesActiveCompletion() {
542 CountContext context;
543 completion.
arm(countCallback,
reinterpret_cast<uintptr_t
>(&context), 43);
545 const bool teardownClaimed = completion.
claimForTeardown(-TransactionError, claim);
548 countCallback,
reinterpret_cast<uintptr_t
>(&context), -TransactionError, duplicate);
549 const bool claimedTwice = completion.
claimForTeardown(-TransactionError, duplicate);
552 check(teardownClaimed && !capturedAfterTeardown &&
553 cancellationAfterTeardown ==
554 UsbHcd::TransferCompletion::CancellationDisposition::DrainPublished &&
555 !claimedTwice && claim.generation == 43 && claim.result == -TransactionError &&
556 claim.reason == UsbHcd::TransferCompletion::Reason::Teardown,
557 "usb-completion-active-teardown",
558 "teardown did not terminalize an accepted hardware obligation once");
560 NOTICE(
"HOSTED-WAIT-TEST: PASS usb-completion-active-teardown");
564bool teardownPreservesCapturedResult() {
568 CountContext context;
569 completion.
arm(countCallback,
reinterpret_cast<uintptr_t
>(&context), 44);
572 const bool teardownClaimed = completion.
claimForTeardown(-TransactionError, claim);
573 const bool ordinaryClaimed = completion.
claimCaptured(duplicate);
576 check(captured && teardownClaimed && !ordinaryClaimed && claim.generation == 44 &&
577 claim.result == 1234 && claim.reason == UsbHcd::TransferCompletion::Reason::Natural,
578 "usb-completion-captured-teardown",
579 "teardown discarded or duplicated a captured hardware result");
581 NOTICE(
"HOSTED-WAIT-TEST: PASS usb-completion-captured-teardown");
585bool cancellationPreservesCapturedResult() {
588 CountContext context;
589 completion.
arm(countCallback,
reinterpret_cast<uintptr_t
>(&context), 45);
593 countCallback,
reinterpret_cast<uintptr_t
>(&context), -TransactionError, claim);
595 const bool passed = check(
596 captured && cancellation == UsbHcd::TransferCompletion::CancellationDisposition::Claimed &&
597 claim.generation == 45 && claim.result == 5678 &&
598 claim.reason == UsbHcd::TransferCompletion::Reason::Natural,
599 "usb-completion-cancel-preserves-natural",
600 "cancellation replaced a hardware result which won the claim race");
603 "HOSTED-WAIT-TEST: PASS "
604 "usb-completion-cancel-preserves-natural");
608bool cancellationRequiresExactIdentity() {
611 CountContext context;
612 completion.
arm(countCallback,
reinterpret_cast<uintptr_t
>(&context), 46);
615 nullptr,
reinterpret_cast<uintptr_t
>(&context), -TransactionError, claim);
616 const auto wrongParameter =
619 countCallback,
reinterpret_cast<uintptr_t
>(&context), -TransactionError, claim);
621 const bool passed = check(
622 wrongCallback == UsbHcd::TransferCompletion::CancellationDisposition::NoMatch &&
623 wrongParameter == UsbHcd::TransferCompletion::CancellationDisposition::NoMatch &&
624 exact == UsbHcd::TransferCompletion::CancellationDisposition::Claimed &&
625 claim.generation == 46,
626 "usb-completion-cancel-identity",
"a stale callback identity claimed a reused transfer slot");
628 NOTICE(
"HOSTED-WAIT-TEST: PASS usb-completion-cancel-identity");
633bool runHostedUsbCallbackDeliveryRegressions() {
634 return pendingRecordCanBeStolen() && runningRecordCanDrainItself() &&
635 anotherThreadWaitsForRunningRecord() && producerWaitsForStolenRunningRecord() &&
636 generationIsPartOfIdentity() && allPendingRecordsCanBeDrained() &&
637 recurringCancellationSuppressesPendingSamples() &&
638 reciprocalSubscriptionCancellationDoesNotDeadlock() &&
639 capturedCompletionHasOnePublisher() && cancellationOwnsCompletion() &&
640 teardownTerminalizesActiveCompletion() && teardownPreservesCapturedResult() &&
641 cancellationPreservesCapturedResult() && cancellationRequiresExactIdentity();
static Scheduler & instance()
MUST_USE_RESULT CancellationDisposition claimCancellation(Callback callback, uintptr_t parameter, ssize_t cancellationResult, Claim &claim)
void arm(Callback callback, uintptr_t parameter, size_t generation)
MUST_USE_RESULT bool claimForTeardown(ssize_t cancellationResult, Claim &claim)
MUST_USE_RESULT bool captureNatural(ssize_t result)
MUST_USE_RESULT bool claimCaptured(Claim &claim)
void pushBack(const T &value)