The Pedigree Project 0.1
Ehci.cc
1/*
2 * Copyright (c) 2008-2014, Pedigree Developers
3 *
4 * Please see the CONTRIB file in the root of the source tree for a full
5 * list of contributors.
6 *
7 * Permission to use, copy, modify, and distribute this software for any
8 * purpose with or without fee is hereby granted, provided that the above
9 * copyright notice and this permission notice appear in all copies.
10 *
11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
18 */
19
20#include "Ehci.h"
21#include "pedigree/kernel/LockGuard.h"
22#include "pedigree/kernel/Log.h"
23#include "pedigree/kernel/Spinlock.h"
24#include "pedigree/kernel/machine/Device.h"
25#include "pedigree/kernel/machine/IrqManager.h"
26#include "pedigree/kernel/machine/Machine.h"
27#include "pedigree/kernel/machine/Pci.h"
28#include "pedigree/kernel/machine/PciFunctionState.h"
29#include "pedigree/kernel/machine/types.h"
30#include "pedigree/kernel/panic.h"
31#include "pedigree/kernel/process/Mutex.h"
32#include "pedigree/kernel/process/TerminationDeferral.h"
33#include "pedigree/kernel/process/Thread.h"
34#include "pedigree/kernel/processor/InterruptHandler.h"
35#include "pedigree/kernel/processor/InterruptManager.h"
36#include "pedigree/kernel/processor/IoBase.h"
37#include "pedigree/kernel/processor/MemoryRegion.h"
38#include "pedigree/kernel/processor/PhysicalMemoryManager.h"
39#include "pedigree/kernel/processor/Processor.h"
40#include "pedigree/kernel/processor/ProcessorInformation.h"
41#include "pedigree/kernel/processor/VirtualAddressSpace.h"
42#include "pedigree/kernel/processor/state_forward.h"
43#include "pedigree/kernel/processor/types.h"
44#include "pedigree/kernel/time/Time.h"
45#include "pedigree/kernel/utilities/ExtensibleBitmap.h"
46#include "pedigree/kernel/utilities/RequestQueue.h"
47#include "pedigree/kernel/utilities/String.h"
48#include "pedigree/kernel/utilities/Vector.h"
49#include "pedigree/kernel/utilities/utility.h"
50
51#include "EhciLegacy.h"
52#include "modules/drivers/common/DmaBuffer.h"
53#include "modules/system/usb/Usb.h"
54#include "modules/system/usb/UsbHub.h"
55
56namespace {
57constexpr size_t EhciHardwarePageBytes = 4096;
58constexpr size_t EhciHardwarePageShift = 12;
59constexpr size_t EhciDescriptorOffsetMask = EhciHardwarePageBytes - 1;
60constexpr size_t EhciQhRegionBytes = 2 * EhciHardwarePageBytes;
61constexpr size_t EhciFrameListOffset = EhciQhRegionBytes;
62constexpr size_t EhciQtdListOffset = EhciFrameListOffset + EhciHardwarePageBytes;
63constexpr size_t EhciDmaRegionBytes = EhciQtdListOffset + EhciHardwarePageBytes;
64constexpr size_t EhciQtdBufferPageCount = 5;
65constexpr size_t EhciQtdBufferWindowBytes = EhciQtdBufferPageCount * EhciHardwarePageBytes;
66constexpr physical_uintptr_t EhciHighestDmaAddress = 0xffffffff;
67static_assert((size_t{1} << EhciHardwarePageShift) == EhciHardwarePageBytes);
68
69struct EhciDmaTransfer {
70 MemoryRegion* region;
71 uintptr_t client;
72 size_t bytes;
73 bool input;
74};
75
76EhciDmaTransfer* dmaTransfer(const Ehci::qTD* qtd) {
77 return reinterpret_cast<EhciDmaTransfer*>(static_cast<uintptr_t>(qtd->pDma));
78}
79
80void releaseDmaTransfer(Ehci::qTD* qtd) {
81 auto* transfer = dmaTransfer(qtd);
82 if (!transfer)
83 return;
84 delete transfer->region;
85 delete transfer;
86 qtd->pDma = 0;
87}
88
89void copyDmaInput(const Ehci::qTD* qtd, ssize_t bytes) {
90 auto* transfer = dmaTransfer(qtd);
91 if (!transfer || !transfer->input || bytes <= 0)
92 return;
93 size_t count = static_cast<size_t>(bytes);
94 if (count > transfer->bytes)
95 count = transfer->bytes;
96 MemoryCopy(reinterpret_cast<void*>(transfer->client), transfer->region->virtualAddress(), count);
97}
98} // namespace
99
100#define INDEX_FROM_QTD(ptr) \
101 (((reinterpret_cast<uintptr_t>((ptr)) & EhciDescriptorOffsetMask) / sizeof(qTD)))
102#define PHYS_QTD(idx) (m_pqTDListPhys + ((idx) * sizeof(qTD)))
103
104static int threadStub(void* p);
105
106static constexpr size_t EhciPollLimit = 100;
107static constexpr uint32_t EhciAsyncScheduleStatus = 0x8000;
108static constexpr uint32_t EhciScheduleStatusMask = 0xc000;
109
110static bool waitForMmioState(IoBase* base, size_t registerOffset, uint32_t mask, uint32_t expected,
111 size_t pollLimit = EhciPollLimit) {
112 expected &= mask;
113 const auto deadline = Time::getTicks() + pollLimit * Time::Multiplier::Millisecond;
114 do {
115 if ((base->read32(registerOffset) & mask) == expected) {
116 return true;
117 }
118 // Teardown must not depend on a timer event being delivered to this thread.
119 Processor::pause();
120 } while (Time::getTicks() < deadline);
121 return (base->read32(registerOffset) & mask) == expected;
122}
123
124Ehci::Ehci(Device* pDev)
125 : UsbHub(pDev),
126 RequestQueue(MakeConstantString("EHCI")),
127 m_pBase(nullptr),
128 m_nOpRegsOffset(0),
129 m_nPorts(0),
130 m_IrqProcessingLock(),
131 m_CompletionDeliveries(),
132 m_pQHList(nullptr),
133 m_pQHListPhys(0),
134 m_pFrameList(nullptr),
135 m_pFrameListPhys(0),
136 m_pqTDList(nullptr),
137 m_pqTDListPhys(0),
138 m_pCurrentQueueTail(0),
139 m_pCurrentQueueHead(0),
140 m_EhciMR("Ehci-MR"),
141 m_DequeueCount(0),
142 m_DequeueStopping(false),
143 m_DequeueThread(),
144 m_SubmissionOperations(),
145 m_CancelOperations(),
146 m_CallbackOperations(),
147 m_IrqId(0),
148 m_InterruptHandlerRegistered(false),
149 m_InterruptClosure(0) {
150 setSpecificType(String("EHCI"));
151}
152
153bool Ehci::initialiseController() {
154 if (TargetInfo::getPageSize() < EhciHardwarePageBytes ||
155 (TargetInfo::getPageSize() % EhciHardwarePageBytes)) {
156 ERROR("USB: EHCI: target pages cannot represent 4 KiB EHCI buffer pages");
157 return false;
158 }
159
160 // Allocate the pages we need
161 if (!PhysicalMemoryManager::instance().allocateRegion(
162 m_EhciMR, DriverDma::pageCountForBytes(EhciDmaRegionBytes),
163 PhysicalMemoryManager::continuous | PhysicalMemoryManager::below4GB,
164 VirtualAddressSpace::KernelMode | VirtualAddressSpace::Write)) {
165 ERROR("USB: EHCI: Couldn't allocate Memory Region!");
166 return false;
167 }
168
169 uintptr_t virtualBase = reinterpret_cast<uintptr_t>(m_EhciMR.virtualAddress());
170 uintptr_t physicalBase = m_EhciMR.physicalAddress();
171 m_pQHList = reinterpret_cast<QH*>(virtualBase);
172 m_pFrameList = reinterpret_cast<uint32_t*>(virtualBase + EhciFrameListOffset);
173 m_pqTDList = reinterpret_cast<qTD*>(virtualBase + EhciQtdListOffset);
174 m_pQHListPhys = physicalBase;
175 m_pFrameListPhys = physicalBase + EhciFrameListOffset;
176 m_pqTDListPhys = physicalBase + EhciQtdListOffset;
177
178 DoubleWordSet(m_pFrameList, 1, 0x400);
179
180#if X86_COMMON
181 auto& pci = PciBus::instance();
182 PciFunctionState::State pciState{};
183 if (!pci.inspectFunction(this, pciState)) {
184 ERROR("EHCI: unsupported inherited PCI function state");
185 return false;
186 }
187 m_FirmwarePciCommand = pciState.command;
188 const uint32_t bar = pciState.bars[0];
189 if ((bar & 1U) || ((bar & 6U) != 0 && (bar & 6U) != 4))
190 return false;
191 uint64_t base = bar & ~15U;
192 if (bar & 4U)
193 base |= uint64_t{pciState.bars[1]} << 32;
194 Device::Address* mapping = nullptr;
195 for (auto* address : addresses())
196 if (address->m_Name == "bar0" && base && !address->m_IsIoSpace && address->m_Address == base)
197 mapping = address;
198 if (!mapping || mapping->m_Size < 0x20 || !pci.updateCommand(this, 0, 2))
199 return false;
200 // Firmware may still need DMA and ownership-change SMIs to complete handoff.
201 mapping->map();
202 m_pBase = mapping->m_Io;
203#else
204 return false;
205#endif
206 if (!m_pBase)
207 return false;
208
209 uint32_t hccapbase = m_pBase->read32(EHCI_CAPLENGTH);
210 uint16_t version = hccapbase >> 16;
211
212 m_nOpRegsOffset = hccapbase & 0xFF;
213 EMIT_IF(UsbVerboseDebug) {
214 NOTICE("EHCI operation registers are at offset " << m_nOpRegsOffset);
215 }
216 if (m_nOpRegsOffset < 0x10 || m_nOpRegsOffset > m_pBase->size() ||
217 m_pBase->size() - m_nOpRegsOffset < EHCI_PORTSC + sizeof(uint32_t)) {
218 // No offset for operational base: this is almost certainly not really
219 // a controller.
220 return false;
221 }
222
223 NOTICE("EHCI controller version " << (version >> 8) << "." << (version & 0xFF));
224
225 // Get structural capabilities to determine the number of physical ports
226 // we have available to us.
227 uint32_t hcsparams = m_pBase->read32(EHCI_HCSPARAMS);
228 m_nPorts = hcsparams & 0xF;
229 if (!UsbHcd::validEhciRootPortCount(m_nPorts)) {
230 ERROR("EHCI: unsupported root-port count " << Dec << m_nPorts << Hex);
231 return false;
232 }
233 EMIT_IF(UsbVerboseDebug) {
234 NOTICE("EHCI controller has " << Dec << m_nPorts << Hex << " physical ports.");
235 }
236
237 uint32_t hccparams = m_pBase->read32(EHCI_HCCPARAMS);
238 uint8_t eecp = (hccparams >> 8) & 0xFF;
239 constexpr uint8_t EecpMinimum = 0x40;
240 constexpr uint8_t EecpMaximum = 0xfc;
241 if (eecp && (eecp < EecpMinimum || eecp > EecpMaximum || (eecp & 0x3))) {
242 ERROR("EHCI: EECP pointer is invalid");
243 return false;
244 }
245
246 EMIT_IF(UsbVerboseDebug) {
247 DEBUG_LOG("EHCI: Host controller " << (hccparams & 1 ? "does" : "does not")
248 << " require 64-bit data structures.");
249 DEBUG_LOG(" Host controller " << (hccparams & 2 ? "does" : "does not")
250 << " allow us to use frame lists with "
251 "anything other than 1024 items in them.");
252 DEBUG_LOG(" Host controller " << (hccparams & 4 ? "does" : "does not")
253 << " support the asynchronous schedule park capability.");
254 DEBUG_LOG(" HCCAPBASE is " << hccapbase);
255 DEBUG_LOG(" HCCPARAMS is " << hccparams);
256 DEBUG_LOG(" HCSPARAMS is " << hcsparams);
257 DEBUG_LOG(" EECP is " << eecp);
258 }
259
260#if X86_COMMON
261 // Pre-OS to OS handoff
262 constexpr size_t EecpSlotCount = (0x100 - EecpMinimum) / 4;
263 size_t capabilityBudget = EecpSlotCount;
264 uint64_t visitedCapabilities = 0;
265 bool foundLegacy = false;
266 while (eecp) {
267 if (!capabilityBudget) {
268 ERROR("EHCI: EECP capability chain exceeded PCI config space");
269 return false;
270 }
271 --capabilityBudget;
272 const size_t capabilitySlot = (eecp - EecpMinimum) / 4;
273 const uint64_t capabilityBit = uint64_t(1) << capabilitySlot;
274 if (visitedCapabilities & capabilityBit) {
275 ERROR("EHCI: cyclic EECP capability chain");
276 return false;
277 }
278 visitedCapabilities |= capabilityBit;
279
280 EMIT_IF(UsbVerboseDebug) {
281 DEBUG_LOG("EHCI: Reading LEGSUP register and checking for BIOS ownership.");
282 }
283 uint32_t legsup = 0;
284 if (!pci.readConfig32(this, eecp, legsup))
285 return false;
286
287 if ((legsup & 0xff) == 1) {
288 if (eecp > 0xf8) {
289 ERROR("EHCI: truncated USB legacy capability");
290 return false;
291 }
292 if (foundLegacy) {
293 ERROR("EHCI: duplicate USB legacy capability");
294 return false;
295 }
296 foundLegacy = true;
297 if (!acquireFirmware(eecp))
298 return false;
299 }
300
301 eecp = (legsup >> 8) & 0xFF; // Zero = "end of list"
302 if (eecp && (eecp < EecpMinimum || eecp > EecpMaximum || (eecp & 0x3))) {
303 ERROR("EHCI: EECP capability chain contains an invalid pointer");
304 return false;
305 }
306 }
307#endif
308
309 m_HardwareOwned = true;
310 m_HardwareTouched = true;
311 m_pBase->write32(0, m_nOpRegsOffset + EHCI_INTR);
312 (void)m_pBase->read32(m_nOpRegsOffset + EHCI_INTR);
313
314 EMIT_IF(UsbVerboseDebug) {
315 DEBUG_LOG("USB: EHCI: disabling running schedules");
316 }
317 // Disable any running schedules gracefully before halting the controller
318 m_pBase->write32(
319 m_pBase->read32(m_nOpRegsOffset + EHCI_CMD) & ~(EHCI_CMD_ASYNCLE | EHCI_CMD_PERIODICLE),
320 m_nOpRegsOffset + EHCI_CMD);
321 if (!waitForMmioState(m_pBase, m_nOpRegsOffset + EHCI_STS, EhciScheduleStatusMask, 0)) {
322 ERROR("EHCI: schedules did not stop within 100 ms");
323 return false;
324 }
325
326 uint32_t status = m_pBase->read32(m_nOpRegsOffset + EHCI_STS);
327 if (!(status & EHCI_STS_HALTED)) {
328 EMIT_IF(UsbVerboseDebug) {
329 DEBUG_LOG("USB: EHCI: pausing controller");
330 }
331 // Must halt the controller, it's not yet halted.
332 m_pBase->write32(m_pBase->read32(m_nOpRegsOffset + EHCI_CMD) & ~EHCI_CMD_RUN,
333 m_nOpRegsOffset + EHCI_CMD);
334 if (!waitForMmioState(m_pBase, m_nOpRegsOffset + EHCI_STS, EHCI_STS_HALTED, EHCI_STS_HALTED)) {
335 ERROR("EHCI: controller did not halt within 100 ms");
336 return false;
337 }
338 }
339
340 EMIT_IF(UsbVerboseDebug) {
341 DEBUG_LOG("USB: EHCI: resetting controller");
342 }
343 // Write host controller reset command and wait for it to complete
344 m_pBase->write32(EHCI_CMD_HCRES, m_nOpRegsOffset + EHCI_CMD);
345 if (!waitForMmioState(m_pBase, m_nOpRegsOffset + EHCI_CMD, EHCI_CMD_HCRES, 0)) {
346 ERROR("EHCI: controller reset did not complete within 100 ms");
347 return false;
348 }
349 EMIT_IF(UsbVerboseDebug) {
350 DEBUG_LOG("USB: EHCI: Reset complete, status: " << m_pBase->read32(m_nOpRegsOffset + EHCI_STS)
351 << ".");
352 }
353
354#if X86_COMMON
355 if (!pci.updateCommand(this, 4, 2 | 0x400) || !pci.disableMessageInterrupts(this, pciState)) {
356 ERROR("EHCI: could not establish masked PCI interrupt state");
357 return false;
358 }
359#endif
360
361 // The queue and every port token must be live before the interrupt source
362 // can publish a port change.
363#if THREADS
364 RequestQueue::initialise();
365 if (getLifecycleState() != RequestQueue::LifecycleState::Accepting) {
366 ERROR("EHCI: request queue did not enter the accepting state");
367 RequestQueue::destroy();
368 return false;
369 }
370 for (size_t i = 0; i < m_nPorts; ++i) {
371 if (!m_PortChanges[i].configure(*this, 0, i)) {
372 ERROR("EHCI: could not configure root-port publication " << i);
373 RequestQueue::destroy();
374 return false;
375 }
376 }
377#endif
378
379 // Do not rely on reset defaults while installing the handler.
380 m_pBase->write32(0, m_nOpRegsOffset + EHCI_INTR);
381 (void)m_pBase->read32(m_nOpRegsOffset + EHCI_INTR);
382
383// Install the IRQ handler
384#if X86_COMMON
385 m_IrqId = Machine::instance().getIrqManager()->registerPciIrqHandler(
386 static_cast<IrqHandler*>(this), this, IrqPolicy::pciIntxThreaded());
387 m_InterruptHandlerRegistered = m_IrqId != 0;
388#else
389 // InterruptManager pointer removal cannot drain a dispatch that already
390 // loaded this handler. Do not advertise teardown safety on that dormant
391 // platform path until it has a synchronous registration API.
392 ERROR("EHCI requires synchronous IRQ handler lifetime management");
393#if THREADS
394 RequestQueue::destroy();
395#endif
396 return false;
397#endif
398 if (!m_InterruptHandlerRegistered) {
399 ERROR("EHCI: could not register interrupt handler");
400#if THREADS
401 RequestQueue::destroy();
402#endif
403 return false;
404 }
405 Machine::instance().getIrqManager()->control(
406 getInterruptNumber(), IrqManager::MitigationThreshold,
407 7500000 / 64); // 58 MB/s (480Mbps) in bytes/s, divided by 64 bytes
408 // maximum per control transfer/IRQ
409
410#if X86_COMMON
411 if (!pci.resourcesUnchanged(this, pciState)) {
412 ERROR("EHCI: PCI resources changed during handoff");
413 return false;
414 }
415#endif
416
417 // Zero the top 64 bits for addresses of EHCI data structures
418 m_pBase->write32(0, m_nOpRegsOffset + EHCI_CTRLDSEG);
419
420 // Write the base address of the periodic frame list - all T-bits are set to
421 // one
422 m_pBase->write32(m_pFrameListPhys, m_nOpRegsOffset + EHCI_PERIODICLP);
423
424 Time::delay(5 * Time::Multiplier::Millisecond);
425
426 // Create a dummy QH and qTD
427 m_QHBitmap.set(0);
428 m_qTDBitmap.set(0);
429 QH* pDummyQH = &m_pQHList[0];
430 qTD* pDummyTD = &m_pqTDList[0];
431 ByteSet(pDummyQH, 0, sizeof(QH));
432 ByteSet(pDummyTD, 0, sizeof(qTD));
433
434 // Configure the dummy TD
435 pDummyTD->bNextInvalid = pDummyTD->bAltNextInvalid = 1;
436
437 // Configure the dummy QH
438 pDummyQH->pNext = m_pQHListPhys >> 5;
439 pDummyQH->nNextType = 1;
440
441 pDummyQH->pQTD = m_pqTDListPhys >> 5;
442 pDummyQH->mult = 1;
443 pDummyQH->hrcl = 1;
444
445 pDummyQH->pMetaData = new QH::MetaData;
446 pDummyQH->pMetaData->pFirstQTD = 0;
447 pDummyQH->pMetaData->pLastQTD = pDummyTD;
448 pDummyQH->pMetaData->pNext = pDummyQH;
449 pDummyQH->pMetaData->pPrev = pDummyQH;
450
451 MemoryCopy(&pDummyQH->overlay, pDummyTD, sizeof(qTD));
452
453 m_pCurrentQueueHead = m_pCurrentQueueTail = pDummyQH;
454
455 // Disable the asynchronous schedule, and wait for it to become disabled
456 m_pBase->write32(m_pBase->read32(m_nOpRegsOffset + EHCI_CMD) & ~EHCI_CMD_ASYNCLE,
457 m_nOpRegsOffset + EHCI_CMD);
458 if (!waitForMmioState(m_pBase, m_nOpRegsOffset + EHCI_STS, EhciAsyncScheduleStatus, 0)) {
459 ERROR("EHCI asynchronous schedule did not stop within 100 ms");
460 return false;
461 }
462
463 // Write the async list head pointer
464 m_pBase->write32(m_pQHListPhys, m_nOpRegsOffset + EHCI_ASYNCLP);
465
466 // Set the desired interrupt threshold (frame list size = 4096 bytes)
467 m_pBase->write32((m_pBase->read32(m_nOpRegsOffset + EHCI_CMD) & ~0xFF0000) | 0x80000,
468 m_nOpRegsOffset + EHCI_CMD);
469
470 FENCE();
471#if X86_COMMON
472 if (!pci.updateCommand(this, 0, 6 | 0x400))
473 return false;
474#endif
475
476 // Turn on the controller
477 m_pBase->write32(m_pBase->read32(m_nOpRegsOffset + EHCI_CMD) | EHCI_CMD_RUN,
478 m_nOpRegsOffset + EHCI_CMD);
479 if (!waitForMmioState(m_pBase, m_nOpRegsOffset + EHCI_STS, EHCI_STS_HALTED, 0)) {
480 ERROR("EHCI controller did not start within 100 ms");
481 return false;
482 }
483
484 m_DequeueThread.adopt(new Thread(Processor::information().getCurrentThread()->getParent(),
485 threadStub, reinterpret_cast<void*>(this)));
486 m_DequeueThread->setName("EHCI dequeue");
487
488#if X86_COMMON
489 if (!pci.updateCommand(this, 0x400, 6))
490 return false;
491#endif
492
493 // The schedules, queue metadata, and completion worker are now live.
494 // PORTCH remains masked until the initial root-port scan below.
495 m_pBase->write32(0x3b, m_nOpRegsOffset + EHCI_INTR);
496 (void)m_pBase->read32(m_nOpRegsOffset + EHCI_INTR);
497
498 // Take over the ports
499 m_pBase->write32(1, m_nOpRegsOffset + EHCI_CFGFLAG);
500
501 // If it's supported, enable the asynchronous park mode with only one
502 // transaction before advancing through the queue.
503 if (hccparams & 4) {
504 m_pBase->write32(m_pBase->read32(m_nOpRegsOffset + EHCI_CMD) | 0x900,
505 m_nOpRegsOffset + EHCI_CMD);
506 }
507
508 // Enable the asynchronous schedule, and wait for it to become enabled
509 m_pBase->write32(m_pBase->read32(m_nOpRegsOffset + EHCI_CMD) | EHCI_CMD_ASYNCLE,
510 m_nOpRegsOffset + EHCI_CMD);
511 if (!waitForMmioState(m_pBase, m_nOpRegsOffset + EHCI_STS, EhciAsyncScheduleStatus,
512 EhciAsyncScheduleStatus)) {
513 ERROR("EHCI asynchronous schedule did not start within 100 ms");
514 return false;
515 }
516
517 // Clear the aggregate before scanning. Any edge after this flush remains
518 // pending for the live publication path when PORTCH is enabled below.
519 m_pBase->write32(EHCI_STS_PORTCH, m_nOpRegsOffset + EHCI_STS);
520 (void)m_pBase->read32(m_nOpRegsOffset + EHCI_STS);
521
522 // Hold recovery until every initially queued root port has had a chance.
523 {
524 LockGuard<Spinlock> startupGuard(m_StartupLock);
525 m_Startup.enableRecovery(m_Handoff.wasBiosOwned());
526 m_InitialPortMask = (1U << m_nPorts) - 1;
527 if (m_InitialPortMask && !m_Startup.begin())
528 return false;
529 }
530
531 // Search for ports with devices and initialise them.
532 for (size_t i = 0; i < m_nPorts; i++) {
533 EMIT_IF(UsbVerboseDebug) {
534 DEBUG_LOG("USB: EHCI: Port " << Dec << i << Hex << " - status initially: "
535 << m_pBase->read32(m_nOpRegsOffset + EHCI_PORTSC + i * 4));
536 }
537 // Check for port power
538 if (!(m_pBase->read32(m_nOpRegsOffset + EHCI_PORTSC + i * 4) & EHCI_PORTSC_PPOW)) {
539 modifyPortControl(m_nOpRegsOffset + EHCI_PORTSC + i * 4, 0, EHCI_PORTSC_PPOW);
540 Time::delay(20 * Time::Multiplier::Millisecond);
541 EMIT_IF(UsbVerboseDebug) {
542 DEBUG_LOG("USB: EHCI: Port " << Dec << i << Hex << " - status after power-up: "
543 << m_pBase->read32(m_nOpRegsOffset + EHCI_PORTSC + i * 4));
544 }
545 }
546
547 // Check for an existing reset on the port and request termination
548 const size_t portRegister = m_nOpRegsOffset + EHCI_PORTSC + i * 4;
549 modifyPortControl(portRegister, EHCI_PORTSC_PRES, 0);
550 if (!waitForMmioState(m_pBase, portRegister, EHCI_PORTSC_PRES, 0)) {
551 ERROR("EHCI: initial reset on port " << Dec << i << Hex
552 << " did not clear within "
553 "100 ms");
554 {
555 StartupActivity failedPort(this);
556 if (failedPort)
557 failedPort.failed();
558 }
559 completeInitialPort(i);
560 continue;
561 }
562
563 constexpr uint32_t ChangeMask = EHCI_PORTSC_CSCH | EHCI_PORTSC_ENCH | EHCI_PORTSC_OCCH;
564 const uint32_t portStatus = m_pBase->read32(portRegister);
565 const uint32_t acknowledgeMask = portStatus & ChangeMask;
566 if (acknowledgeMask) {
567 m_pBase->write32(UsbHcd::selectiveW1cValue(portStatus, ChangeMask, acknowledgeMask),
568 portRegister);
569 (void)m_pBase->read32(portRegister);
570 }
571
572#if THREADS
573 // Controller discovery runs inside the device-tree walk. Defer topology
574 // mutation until that walk releases its non-recursive tree lock.
575 const auto observation = m_PortChanges[i].observe();
576 if (!UsbHcd::PortChangeRequest::canAcknowledge(observation.result)) {
577 panic("EHCI could not publish its initial root-port state");
578 }
579 m_PortChanges[i].acknowledge(observation.generation);
580#else
581 executeRequest(i);
582#endif
583 }
584
585#if THREADS
586 m_pBase->write32(0x3f, m_nOpRegsOffset + EHCI_INTR);
587#else
588 // Enumerating a device can block and must never run in interrupt context.
589 m_pBase->write32(0x3b, m_nOpRegsOffset + EHCI_INTR);
590#endif
591
592 return true;
593}
594
595Ehci::~Ehci() {
596 m_RecoveryStopping = true;
597 m_RecoveryWake.release();
598 m_RecoveryThread.join();
599 shutdownController();
600 returnToFirmware();
601}
602
603void Ehci::shutdownController() {
604 if (m_ControllerStopped)
605 return;
606 m_ControllerStopped = true;
607 {
608 LockGuard<Spinlock> startupGuard(m_StartupLock);
609 m_Startup.close();
610 }
611 // Quiesce only the port producer first. Transfer completion IRQs and the
612 // dequeue worker must remain live while an active port request drains.
613 {
614 LockGuard<IrqProcessingLock> transactionGuard(m_IrqProcessingLock);
615 m_InterruptClosure = 1;
616 if (m_HardwareOwned) {
617 const uint32_t interrupts = m_pBase->read32(m_nOpRegsOffset + EHCI_INTR);
618 m_pBase->write32(interrupts & ~EHCI_STS_PORTCH, m_nOpRegsOffset + EHCI_INTR);
619 (void)m_pBase->read32(m_nOpRegsOffset + EHCI_INTR);
620 }
621 }
622
623 // The PORTCH mask and IRQ serialization above close and drain observe().
624 for (size_t i = 0; i < m_nPorts; ++i) {
625 m_PortChanges[i].stopAfterQuiesce();
626 }
627 RequestQueue::destroy();
628
629 // Enumeration is quiesced, while transfer cancellation and DMA are still
630 // live for class-driver destructors.
631 disconnectAllDevices();
632
633 // Port enumeration has drained, so no internal producer needs to construct
634 // another transfer while the controller is being halted.
635 m_SubmissionOperations.closeAndWait();
636
637 {
638 LockGuard<Mutex> controllerGuard(m_Mutex);
639 {
640 LockGuard<IrqProcessingLock> transactionGuard(m_IrqProcessingLock);
641 m_InterruptClosure = 2;
642 if (m_HardwareOwned) {
643 m_pBase->write32(0, m_nOpRegsOffset + EHCI_INTR);
644 (void)m_pBase->read32(m_nOpRegsOffset + EHCI_INTR);
645 const uint32_t pending = m_pBase->read32(m_nOpRegsOffset + EHCI_STS) & 0x3f;
646 if (pending) {
647 m_pBase->write32(pending, m_nOpRegsOffset + EHCI_STS);
648 (void)m_pBase->read32(m_nOpRegsOffset + EHCI_STS);
649 }
650 }
651 m_CallbackOperations.close();
652 }
653
654 if (m_HardwareOwned) {
655 const uint32_t command = m_pBase->read32(m_nOpRegsOffset + EHCI_CMD);
656 m_pBase->write32(command & ~EHCI_CMD_RUN, m_nOpRegsOffset + EHCI_CMD);
657 (void)m_pBase->read32(m_nOpRegsOffset + EHCI_CMD);
658
659 if (!waitForMmioState(m_pBase, m_nOpRegsOffset + EHCI_STS, EHCI_STS_HALTED,
660 EHCI_STS_HALTED)) {
661 panic("EHCI teardown could not establish the DMA halt boundary");
662 }
663
664 const uint32_t pending = m_pBase->read32(m_nOpRegsOffset + EHCI_STS) & 0x3f;
665 if (pending) {
666 m_pBase->write32(pending, m_nOpRegsOffset + EHCI_STS);
667 (void)m_pBase->read32(m_nOpRegsOffset + EHCI_STS);
668 }
669 m_pBase->write32(0, m_nOpRegsOffset + EHCI_ASYNCLP);
670 m_pBase->write32(0, m_nOpRegsOffset + EHCI_PERIODICLP);
671 (void)m_pBase->read32(m_nOpRegsOffset + EHCI_PERIODICLP);
672 }
673 }
674#if X86_COMMON
675 if (m_HardwareOwned && !PciBus::instance().updateCommand(this, 4, 0x400))
676 panic("EHCI teardown could not disable PCI DMA and INTx");
677 if (m_InterruptHandlerRegistered) {
678 if (!Machine::instance().getIrqManager()->unregisterHandler(m_IrqId,
679 static_cast<IrqHandler*>(this))) {
680 panic(
681 "EHCI teardown could not synchronously unregister its IRQ "
682 "callback");
683 }
684 m_IrqId = 0;
685 m_InterruptHandlerRegistered = false;
686 }
687#endif
688 m_CallbackOperations.wait();
689 m_DequeueStopping = true;
690 m_DequeueCount.release();
691 m_DequeueThread.join();
692
693 // The IRQ and dequeue worker are the only ordinary callback publishers.
694 // Cancellation stays open while their callbacks drain and while teardown
695 // callbacks are captured below.
696 (void)m_CompletionDeliveries.drainAll();
697
698 List<UsbHcd::CallbackDeliveryQueue::Record*> completions;
699 {
700 LockGuard<Mutex> controllerGuard(m_Mutex);
701 LockGuard<IrqProcessingLock> transactionGuard(m_IrqProcessingLock);
702
703 if (m_pFrameList) {
704 DoubleWordSet(m_pFrameList, 1, 0x400);
705 for (size_t i = 0; i < 1024; ++i) {
706 if (m_FrameBitmap.test(i))
707 m_FrameBitmap.clear(i);
708 }
709 }
710
711 if (m_pQHList) {
712 constexpr size_t QhCount = EhciQhRegionBytes / sizeof(QH);
713 for (size_t i = 1; i < QhCount; ++i) {
714 if (!m_QHBitmap.test(i))
715 continue;
716
717 QH* qh = &m_pQHList[i];
718 if (!qh->pMetaData) {
719 ByteSet(qh, 0, sizeof(QH));
720 m_QHBitmap.clear(i);
721 continue;
722 }
723
724 if (qh->pMetaData->bPeriodic ||
725 qh->pMetaData->completion.state() == UsbHcd::TransferCompletion::State::Idle) {
726 reclaimQhLocked(i);
727 continue;
728 }
729
730 UsbHcd::TransferCompletion::Claim claim;
731 if (qh->pMetaData->completion.claimForTeardown(-TransactionError, claim)) {
732 captureCompletionLocked(i, qh, claim, completions);
733 }
734 }
735
736 if (m_QHBitmap.test(0)) {
737 QH* dummy = &m_pQHList[0];
738 LockGuard<Spinlock> queueGuard(m_QueueListChangeLock);
739 dummy->pNext = m_pQHListPhys >> 5;
740 dummy->nNextType = 1;
741 if (dummy->pMetaData) {
742 dummy->pMetaData->pNext = dummy;
743 dummy->pMetaData->pPrev = dummy;
744 }
745 m_pCurrentQueueHead = m_pCurrentQueueTail = dummy;
746 }
747 }
748
749 if (completions.count())
750 m_CompletionDeliveries.publish(completions);
751 }
752
753 while (completions.count()) {
754 auto* completion = completions.popFront();
755 m_CompletionDeliveries.deliver(completion);
756 }
757
758 // A cancellation may have stolen a teardown record. Keep cancellation
759 // admission open until every such callback, including nested drains, exits.
760 (void)m_CompletionDeliveries.drainAll();
761 m_CancelOperations.closeAndWait();
762 (void)m_CompletionDeliveries.drainAll();
763 assert(m_CompletionDeliveries.empty());
764
765 {
766 LockGuard<Mutex> controllerGuard(m_Mutex);
767 if (m_pQHList) {
768 constexpr size_t QhCount = EhciQhRegionBytes / sizeof(QH);
769 for (size_t i = 1; i < QhCount; ++i)
770 assert(!m_QHBitmap.test(i));
771
772 if (m_QHBitmap.test(0)) {
773 delete m_pQHList[0].pMetaData;
774 ByteSet(&m_pQHList[0], 0, sizeof(QH));
775 m_QHBitmap.clear(0);
776 }
777 }
778
779 if (m_pqTDList) {
780 constexpr size_t QtdCount = EhciHardwarePageBytes / sizeof(qTD);
781 for (size_t i = 1; i < QtdCount; ++i)
782 assert(!m_qTDBitmap.test(i));
783 if (m_qTDBitmap.test(0)) {
784 ByteSet(&m_pqTDList[0], 0, sizeof(qTD));
785 m_qTDBitmap.clear(0);
786 }
787 }
788
789 for (size_t i = 0; i < 1024; ++i)
790 assert(!m_FrameBitmap.test(i));
791 m_pCurrentQueueHead = nullptr;
792 m_pCurrentQueueTail = nullptr;
793 }
794 m_HardwareOwned = false;
795}
796
797static int threadStub(void* p) {
798 Ehci* pEhci = reinterpret_cast<Ehci*>(p);
799 pEhci->doDequeue();
800 return 0;
801}
802
803namespace {
804struct EhciCompletionCleanup {
805 Ehci* controller;
806 size_t transaction;
807 size_t generation;
808};
809} // namespace
810
811void Ehci::releaseQtdChainLocked(QH* qh) {
812 if (!qh || !qh->pMetaData || !qh->pMetaData->pFirstQTD)
813 return;
814
815 constexpr size_t QtdCount = EhciHardwarePageBytes / sizeof(qTD);
816 size_t qtdIndex = INDEX_FROM_QTD(qh->pMetaData->pFirstQTD);
817 size_t budget = QtdCount;
818 bool foundLast = false;
819 while (budget) {
820 --budget;
821 if (qtdIndex >= QtdCount)
822 panic("EHCI: invalid qTD index during reclamation");
823
824 qTD* qtd = &m_pqTDList[qtdIndex];
825 const bool last = qtd->bNextInvalid;
826 size_t nextIndex = 0;
827 if (!last)
828 nextIndex = ((qtd->pNext << 5) & EhciDescriptorOffsetMask) / sizeof(qTD);
829
830 if (m_qTDBitmap.test(qtdIndex))
831 m_qTDBitmap.clear(qtdIndex);
832 releaseDmaTransfer(qtd);
833 ByteSet(qtd, 0, sizeof(qTD));
834
835 if (last) {
836 foundLast = true;
837 break;
838 }
839 if (nextIndex == qtdIndex)
840 panic("EHCI: circular qTD chain during reclamation");
841 qtdIndex = nextIndex;
842 }
843 if (!foundLast)
844 panic("EHCI: qTD reclamation exceeded its descriptor budget");
845
846 qh->pMetaData->pFirstQTD = nullptr;
847 qh->pMetaData->pLastQTD = nullptr;
848 qh->pQTD = 0;
849 ByteSet(&qh->overlay, 0, sizeof(qTD));
850}
851
852void Ehci::reclaimQhLocked(size_t transaction) {
853 if (!m_pQHList || !m_QHBitmap.test(transaction))
854 return;
855
856 QH* qh = &m_pQHList[transaction];
857 if (qh->pMetaData) {
858 releaseQtdChainLocked(qh);
859 delete qh->pMetaData;
860 }
861 ByteSet(qh, 0, sizeof(QH));
862 m_QHBitmap.clear(transaction);
863}
864
865void Ehci::captureCompletionLocked(size_t transaction, QH* qh,
866 const UsbHcd::TransferCompletion::Claim& claim,
867 List<UsbHcd::CallbackDeliveryQueue::Record*>& completions) {
868 assert(qh && qh->pMetaData && !qh->pMetaData->bPeriodic);
869 assert(claim.generation == qh->pMetaData->completion.generation());
870
871 releaseQtdChainLocked(qh);
872 qh->pMetaData->bIgnore = true;
873 auto* cleanup = new EhciCompletionCleanup{this, transaction, claim.generation};
874 completions.pushBack(m_CompletionDeliveries.create({transaction, claim.generation},
875 claim.callback, claim.parameter, claim.result,
876 finishDeferredCompletion, cleanup));
877}
878
879void Ehci::finishDeferredCompletion(void* context) {
880 auto* cleanup = reinterpret_cast<EhciCompletionCleanup*>(context);
881 Ehci* controller = cleanup->controller;
882 {
883 LockGuard<Mutex> guard(controller->m_Mutex);
884 const size_t transaction = cleanup->transaction;
885 if (controller->m_QHBitmap.test(transaction)) {
886 QH* qh = &controller->m_pQHList[transaction];
887 if (qh->pMetaData && qh->pMetaData->completion.generation() == cleanup->generation) {
888 controller->reclaimQhLocked(transaction);
889 }
890 }
891 }
892 delete cleanup;
893}
894
895void Ehci::rebuildPeriodicScheduleLocked() {
896 constexpr size_t Count = EhciQhRegionBytes / sizeof(QH);
897 for (size_t frame = 0; frame < 1024; ++frame) {
898 m_pFrameList[frame] = 1;
899 QH* previous = nullptr;
900 // Power-of-two intervals make each slower QH's successor identical in
901 // every frame containing it: faster queues can be shared by all branches.
902 for (size_t interval = 1024; interval; interval /= 2) {
903 if (frame % interval)
904 continue;
905 for (size_t i = 1; i < Count; ++i) {
906 QH* qh = &m_pQHList[i];
907 if (!m_QHBitmap.test(i) || !qh->pMetaData || !qh->pMetaData->pCallback ||
908 !qh->pMetaData->bPeriodic || qh->pMetaData->periodicInterval != interval)
909 continue;
910 const uint32_t address = m_pQHListPhys + i * sizeof(QH);
911 if (previous) {
912 previous->pNext = address >> 5;
913 previous->nNextType = 1;
914 previous->bNextInvalid = 0;
915 } else
916 m_pFrameList[frame] = address | 2U;
917 qh->bNextInvalid = 1;
918 previous = qh;
919 }
920 }
921 }
922 FENCE();
923}
924void Ehci::rearmPeriodicCompletion(void* context) {
925 auto* rearm = static_cast<EhciCompletionCleanup*>(context);
926 Ehci* controller = rearm->controller;
927 LockGuard<Mutex> lock(controller->m_Mutex);
928 LockGuard<IrqProcessingLock> irqLock(controller->m_IrqProcessingLock);
929 if (!controller->m_QHBitmap.test(rearm->transaction) || controller->m_InterruptClosure >= 2)
930 return;
931 QH* qh = &controller->m_pQHList[rearm->transaction];
932 auto* metadata = qh->pMetaData;
933 if (!metadata || !metadata->pCallback || metadata->periodicGeneration != rearm->generation)
934 return;
935 qTD next = metadata->periodicTemplate;
936 next.nStatus = 0;
937 *metadata->pFirstQTD = next;
938 MemoryCopy(&qh->overlay, metadata->pFirstQTD, sizeof(qTD));
939 metadata->periodicPending = false;
940 FENCE();
941 metadata->pFirstQTD->nStatus = 0x80;
942 FENCE();
943 qh->overlay.nStatus = 0x80;
944}
945void Ehci::destroyPeriodicCompletion(void* context) {
946 delete static_cast<EhciCompletionCleanup*>(context);
947}
948
949void Ehci::doDequeue() {
950 TerminationDeferral workerLifetime;
951 while (true) {
952 const bool woken = m_DequeueCount.acquireForCompletion();
953 (void)woken;
954 if (m_DequeueStopping)
955 return;
956
957 List<UsbHcd::CallbackDeliveryQueue::Record*> completions;
958 {
959 // Absolutely cannot have queue insertions during a dequeue.
960 LockGuard<Mutex> guard(m_Mutex);
961 LockGuard<IrqProcessingLock> transactionGuard(m_IrqProcessingLock);
962
963 for (size_t i = 1; i < EhciQhRegionBytes / sizeof(QH); i++) {
964 if (!m_QHBitmap.test(i))
965 continue;
966
967 QH* pQH = &m_pQHList[i];
968
969 // Is this QH valid?
970 if (!pQH->pMetaData) {
971 EMIT_IF(UsbVerboseDebug) {
972 DEBUG_LOG("Not performing dequeue on QH #" << Dec << i << Hex
973 << " as it's not even initialised.");
974 }
975 continue;
976 }
977
978 // Is this QH even linked!?
979 if (!pQH->pMetaData->bIgnore) {
980 EMIT_IF(UsbVerboseDebug) {
981 DEBUG_LOG("Not performing dequeue on QH #" << Dec << i << Hex
982 << " as it's still active.");
983 }
984 continue;
985 }
986
987 if (pQH->pMetaData->bPeriodic)
988 continue;
989
990 UsbHcd::TransferCompletion::Claim claim;
991 if (!pQH->pMetaData->completion.claimCaptured(claim))
992 continue;
993 captureCompletionLocked(i, pQH, claim, completions);
994
995 EMIT_IF(UsbVerboseDebug) {
996 DEBUG_LOG("Dequeue for QH #" << Dec << i << Hex << ".");
997 }
998 }
999
1000 if (completions.count())
1001 m_CompletionDeliveries.publish(completions);
1002 }
1003
1004 while (completions.count()) {
1005 auto* completion = completions.popFront();
1006 m_CompletionDeliveries.deliver(completion);
1007 }
1008 }
1009}
1010
1011#if X86_COMMON
1012IrqDisposition Ehci::irq(irq_id_t number)
1013#else
1014void Ehci::interrupt(size_t number, InterruptState& state)
1015#endif
1016{
1017 (void)number;
1018#if !X86_COMMON
1019 (void)state;
1020#endif
1021
1022 OperationBarrier::Lease callback;
1023 if (!m_CallbackOperations.tryAcquire(callback)) {
1024 return
1025#if X86_COMMON
1026 IrqDisposition::Quiesced
1027#endif
1028 ;
1029 }
1030#if X86_COMMON
1031 List<UsbHcd::CallbackDeliveryQueue::Record*> completions;
1032 {
1033#endif
1034 LockGuard<IrqProcessingLock> transactionGuard(m_IrqProcessingLock);
1035
1036 if (m_InterruptClosure == 2) {
1037 return
1038#if X86_COMMON
1039 IrqDisposition::Quiesced
1040#endif
1041 ;
1042 }
1043
1044 /*
1045 uint32_t pciStatus = PciBus::instance().readConfigSpace(this, 1) >> 16;
1046 if(!(pciStatus & 8))
1047 {
1048 NOTICE_NOLOCK("EHCI: IRQ fired, but not for us [" << pciStatus <<
1049 "]"); return #if X86_COMMON false #endif
1050 ;
1051 }
1052 */
1053
1054 uint32_t nStatus =
1055 m_pBase->read32(m_nOpRegsOffset + EHCI_STS) & m_pBase->read32(m_nOpRegsOffset + EHCI_INTR);
1056
1057 if (!nStatus) {
1058 EMIT_IF(UsbVerboseDebug) {
1059 DEBUG_LOG_NOLOCK("EHCI: shared IRQ with no pending controller cause");
1060 }
1061 return
1062#if X86_COMMON
1063 IrqDisposition::NotHandled // Shared IRQ: another device
1064#endif
1065 ;
1066 }
1067
1068 // Clear and flush the aggregate before scanning. A later edge will
1069 // relatch PORTCH instead of being erased after its port was already
1070 // scanned.
1071#if THREADS
1072 if (nStatus & EHCI_STS_PORTCH) {
1073 m_pBase->write32(EHCI_STS_PORTCH, m_nOpRegsOffset + EHCI_STS);
1074 (void)m_pBase->read32(m_nOpRegsOffset + EHCI_STS);
1075 }
1076#endif
1077
1078 // ACK non-port causes early.
1079 const uint32_t immediateStatus = nStatus & ~EHCI_STS_PORTCH;
1080 if (immediateStatus) {
1081 m_pBase->write32(immediateStatus, m_nOpRegsOffset + EHCI_STS);
1082 (void)m_pBase->read32(m_nOpRegsOffset + EHCI_STS);
1083 }
1084
1085 if (nStatus & (EHCI_STS_SYSERR | EHCI_STS_ERR)) {
1086 NOTICE_NOLOCK("EHCI: Unusual IRQ, status is " << nStatus);
1087 }
1088
1089 EMIT_IF(UsbVerboseDebug) {
1090 DEBUG_LOG_NOLOCK("EHCI IRQ " << nStatus);
1091 }
1092#if THREADS
1093 if (nStatus & EHCI_STS_PORTCH) {
1094 constexpr uint32_t ChangeMask = EHCI_PORTSC_CSCH | EHCI_PORTSC_ENCH | EHCI_PORTSC_OCCH;
1095 for (size_t i = 0; i < m_nPorts; i++) {
1096 const size_t portRegister = m_nOpRegsOffset + EHCI_PORTSC + i * 4;
1097 const uint32_t portStatus = m_pBase->read32(portRegister);
1098 uint32_t acknowledgeMask = portStatus & (EHCI_PORTSC_ENCH | EHCI_PORTSC_OCCH);
1099
1100 if (portStatus & EHCI_PORTSC_CSCH) {
1101 if (deferConnectionChangeIfSuppressed(i)) {
1102 acknowledgeMask |= EHCI_PORTSC_CSCH;
1103 } else {
1104 const auto observation = m_PortChanges[i].observe();
1105 const bool accepted = UsbHcd::PortChangeRequest::canAcknowledge(observation.result);
1106 assert(accepted);
1107 if (!accepted) {
1108 // Queue acceptance is established before PORTCH is
1109 // enabled and is retained until after PORTCH
1110 // drains. Preserve CSC and fail closed if that
1111 // invariant ever regresses in a non-asserting
1112 // build.
1113 m_InterruptClosure = 1;
1114 const uint32_t interrupts = m_pBase->read32(m_nOpRegsOffset + EHCI_INTR);
1115 m_pBase->write32(interrupts & ~EHCI_STS_PORTCH, m_nOpRegsOffset + EHCI_INTR);
1116 (void)m_pBase->read32(m_nOpRegsOffset + EHCI_INTR);
1117 continue;
1118 }
1119
1120 acknowledgeMask |= EHCI_PORTSC_CSCH;
1121 m_DeferredPortChanges.defer(i, observation.generation);
1122 }
1123 }
1124
1125 if (acknowledgeMask) {
1126 m_pBase->write32(UsbHcd::selectiveW1cValue(portStatus, ChangeMask, acknowledgeMask),
1127 portRegister);
1128 // Flush posted MMIO before allowing the worker to sample.
1129 (void)m_pBase->read32(portRegister);
1130 }
1131 }
1132
1133 for (size_t i = 0; i < m_nPorts; ++i) {
1134 const size_t generation = m_DeferredPortChanges.release(i);
1135 if (generation) {
1136 m_PortChanges[i].acknowledge(generation);
1137 }
1138 }
1139 }
1140#endif
1141
1142 // Because there's no IOC for *every* transfer, we need to handle errors
1143 // that occur before the last transfer. These will create an error
1144 // status only.
1145 if (nStatus & (EHCI_STS_INT | EHCI_STS_ERR)) {
1146 for (size_t i = 1; i < EhciQhRegionBytes / sizeof(QH); i++) {
1147 if (!m_QHBitmap.test(i))
1148 continue;
1149
1150 QH* pQH = &m_pQHList[i];
1151 if (!pQH->pMetaData) // This QH isn't actually ready to be
1152 // handled yet.
1153 continue;
1154 if (!pQH->pMetaData->bPeriodic && !(pQH->pMetaData->pPrev && pQH->pMetaData->pNext))
1155 continue;
1156 if (pQH->pMetaData->bIgnore || pQH->pMetaData->periodicPending)
1157 continue;
1158 if (!(pQH->pMetaData->pFirstQTD && pQH->pMetaData->pLastQTD))
1159 continue;
1160
1161 bool bPeriodic = pQH->pMetaData->bPeriodic;
1162
1163 size_t nQTDIndex = INDEX_FROM_QTD(pQH->pMetaData->pFirstQTD);
1164 constexpr size_t QtdCount = EhciHardwarePageBytes / sizeof(qTD);
1165 size_t qtdBudget = QtdCount;
1166 while (qtdBudget) {
1167 --qtdBudget;
1168 if (nQTDIndex >= QtdCount) {
1169 ERROR_NOLOCK("EHCI: QH #" << Dec << i << Hex << " has an out-of-range qTD pointer");
1170 break;
1171 }
1172
1173 qTD* pqTD = &m_pqTDList[nQTDIndex];
1174
1175 if (!(pqTD->nStatus & 0x80) && !pqTD->processed) {
1176 pqTD->processed = true;
1177 ssize_t nResult;
1178 if (pqTD->nStatus & 0x7c) {
1179 EMIT_IF(UsbVerboseDebug) {
1180 ERROR_NOLOCK(((nStatus & EHCI_STS_ERR) ? "USB" : "qTD") << " ERROR!");
1181 ERROR_NOLOCK("qTD Status: " << pqTD->nStatus
1182 << " [overlay status=" << pQH->overlay.nStatus << "]");
1183 ERROR_NOLOCK("qTD Error Counter: " << pqTD->nErr << " [overlay counter="
1184 << pQH->overlay.nErr << "]");
1185 ERROR_NOLOCK("QH NAK counter: " << pqTD->res1
1186 << " [overlay count=" << pQH->overlay.res1 << "]");
1187 ERROR_NOLOCK("qTD PID: " << pqTD->nPid << ".");
1188 }
1189 nResult = -pqTD->getError();
1190 } else {
1191 nResult = pqTD->nBufferSize - pqTD->nBytes;
1192 copyDmaInput(pqTD, nResult);
1193 pQH->pMetaData->nTotalBytes += nResult;
1194 }
1195 EMIT_IF(UsbVerboseDebug) {
1196 DEBUG_LOG_NOLOCK(
1197 "qTD #" << Dec << nQTDIndex << Hex << " [from QH #" << Dec << i << Hex
1198 << "] DONE: " << Dec << pQH->nAddress << ":" << pQH->nEndpoint << " "
1199 << (pqTD->nPid == 0
1200 ? "OUT"
1201 : (pqTD->nPid == 1 ? "IN" : (pqTD->nPid == 2 ? "SETUP" : "")))
1202 << " " << nResult << Hex);
1203 }
1204
1205 // Last qTD or error condition?
1206 const bool shortIn =
1207 pQH->nEndpoint && pqTD->nPid == 1 && nResult >= 0 && nResult < pqTD->nBufferSize;
1208 if ((nResult < 0) || shortIn || (pqTD == pQH->pMetaData->pLastQTD)) {
1209 const ssize_t completionResult = nResult < 0 ? nResult : pQH->pMetaData->nTotalBytes;
1210 const bool ownsCompletion =
1211 bPeriodic || pQH->pMetaData->completion.captureNatural(completionResult);
1212
1213 if (!bPeriodic && ownsCompletion) {
1214 // Ensure the list doesn't change as we modify
1215 // it
1216 m_QueueListChangeLock.acquire(); // Atomic operation
1217
1218 // Was the reclaim head bit set?
1219 if (pQH->hrcl)
1220 pQH->pMetaData->pNext->hrcl = 1; // Make sure there's always a
1221 // reclaim head
1222
1223 // This queue head is done, dequeue.
1224 QH* pPrev = pQH->pMetaData->pPrev;
1225 QH* pNext = pQH->pMetaData->pNext;
1226
1227 // Main non-hardware linked list update
1228 pPrev->pMetaData->pNext = pNext;
1229 pNext->pMetaData->pPrev = pPrev;
1230
1231 // Hardware linked list update
1232 pPrev->pNext = pQH->pNext;
1233
1234 // Update the tail pointer if we need to
1235 if (pQH == m_pCurrentQueueTail) {
1236 m_pCurrentQueueTail = pPrev;
1237 }
1238
1239 // Interrupt on Async Advance Doorbell - will
1240 // run the dequeue thread to clear bits in the
1241 // QH and qTD bitmaps
1242 size_t cmdReg = m_pBase->read32(m_nOpRegsOffset + EHCI_CMD);
1243 m_pBase->write32(cmdReg | (1 << 6), m_nOpRegsOffset + EHCI_CMD);
1244
1245 // Now ready for dequeue.
1246 pQH->pMetaData->bIgnore = true;
1247
1248 m_QueueListChangeLock.release();
1249 }
1250
1251 if (bPeriodic && pQH->pMetaData->pCallback) {
1252#if X86_COMMON
1253 pQH->pMetaData->periodicPending = true;
1254 pQH->pMetaData->periodicTemplate.bDataToggle = pQH->overlay.bDataToggle;
1255 pQH->pMetaData->nTotalBytes = 0;
1256 auto* rearm =
1257 new EhciCompletionCleanup{this, i, pQH->pMetaData->periodicGeneration};
1258 completions.pushBack(m_CompletionDeliveries.create(
1259 {i, m_CompletionDeliveries.nextGeneration(),
1260 pQH->pMetaData->periodicGeneration},
1261 pQH->pMetaData->pCallback, pQH->pMetaData->pParam, completionResult,
1262 rearmPeriodicCompletion, rearm, destroyPeriodicCompletion, rearm));
1263#else
1264 pQH->pMetaData->pCallback(pQH->pMetaData->pParam, completionResult);
1265#endif
1266 }
1267 }
1268 }
1269
1270 size_t oldIndex = nQTDIndex;
1271
1272 if (pqTD->bNextInvalid)
1273 break;
1274 else
1275 nQTDIndex = ((pqTD->pNext << 5) & EhciDescriptorOffsetMask) / sizeof(qTD);
1276
1277 if (nQTDIndex == oldIndex) {
1278 ERROR_NOLOCK("EHCI: QH #" << Dec << i << Hex
1279 << "'s qTD list is invalid - circular reference!");
1280 break;
1281 } else if (pqTD->pNext == 0) {
1282 ERROR_NOLOCK("EHCI: QH #" << Dec << i << Hex
1283 << "'s qTD list is invalid - null pNext "
1284 "pointer (and T bit not set)!");
1285 break;
1286 }
1287
1288 if (!qtdBudget) {
1289 ERROR_NOLOCK("EHCI: QH #" << Dec << i << Hex << " exceeded the qTD scan budget");
1290 break;
1291 }
1292 }
1293 }
1294 }
1295
1296 if (nStatus & EHCI_STS_ASYNCADVANCE) {
1297 m_DequeueCount.release();
1298 }
1299
1300#if X86_COMMON
1301 if (completions.count())
1302 m_CompletionDeliveries.publish(completions);
1303 }
1304
1305 while (completions.count()) {
1306 auto* completion = completions.popFront();
1307 m_CompletionDeliveries.deliver(completion);
1308 }
1309 return IrqDisposition::Handled;
1310#endif
1311}
1312
1313void Ehci::addTransferToTransaction(uintptr_t nTransaction, bool bToggle, UsbPid pid,
1314 uintptr_t pBuffer, size_t nBytes) {
1315 OperationBarrier::Lease submission;
1316 if (!m_SubmissionOperations.tryAcquire(submission))
1317 return;
1318 addTransferToTransactionAdmitted(nTransaction, bToggle, pid, pBuffer, nBytes);
1319}
1320
1321void Ehci::addTransferToTransactionAdmitted(uintptr_t nTransaction, bool bToggle, UsbPid pid,
1322 uintptr_t pBuffer, size_t nBytes) {
1323 LockGuard<Mutex> guard(m_Mutex);
1324 constexpr size_t QhCount = EhciQhRegionBytes / sizeof(QH);
1325 if (nTransaction >= QhCount || !m_QHBitmap.test(nTransaction)) {
1326 ERROR("EHCI: addTransferToTransaction: invalid transaction");
1327 return;
1328 }
1329
1330 QH* pQH = &m_pQHList[nTransaction];
1331 if (!pQH->pMetaData || pQH->pMetaData->bBuildFailed ||
1332 pQH->pMetaData->completion.state() != UsbHcd::TransferCompletion::State::Idle) {
1333 ERROR("EHCI: addTransferToTransaction: transaction is not building");
1334 return;
1335 }
1336
1337 qTD* last = pQH->pMetaData->pLastQTD;
1338 if (last && !last->pDma && pid != UsbPidSetup && nBytes &&
1339 last->nPid == (pid == UsbPidIn ? 1U : 0U) && last->nBufferSize &&
1340 !(last->nBufferSize % pQH->nMaxPacketSize) &&
1341 last->nOffset + last->nBufferSize + nBytes <= EhciHardwarePageBytes) {
1342 physical_uintptr_t physical = 0;
1343 auto& addressSpace = Processor::information().getVirtualAddressSpace();
1344 if (DriverDma::virtualToPhysical(addressSpace, pBuffer, physical) &&
1345 physical ==
1346 (static_cast<uintptr_t>(last->pPage0) << 12) + last->nOffset + last->nBufferSize) {
1347 last->nBytes += nBytes;
1348 last->nBufferSize += nBytes;
1349 if (last == pQH->pMetaData->pFirstQTD)
1350 MemoryCopy(&pQH->overlay, last, sizeof(qTD));
1351 return;
1352 }
1353 }
1354 const size_t nIndex = m_qTDBitmap.getFirstClear();
1355 if (nIndex >= (EhciHardwarePageBytes / sizeof(qTD))) {
1356 ERROR("USB: EHCI: qTD space full");
1357 pQH->pMetaData->bBuildFailed = true;
1358 return;
1359 }
1360 m_qTDBitmap.set(nIndex);
1361
1362 // Grab the qTD pointer we're going to set up now
1363 qTD* pqTD = &m_pqTDList[nIndex];
1364 ByteSet(pqTD, 0, sizeof(qTD));
1365
1366 // There's nothing after us for now
1367 pqTD->bNextInvalid = 1;
1368 pqTD->bAltNextInvalid = 1;
1369
1370 // PID for the transfer
1371 switch (pid) {
1372 case UsbPidOut:
1373 pqTD->nPid = 0;
1374 break;
1375 case UsbPidIn:
1376 pqTD->nPid = 1;
1377 break;
1378 case UsbPidSetup:
1379 pqTD->nPid = 2;
1380 break;
1381 default:
1382 pqTD->nPid = 3;
1383 };
1384
1385 // Active, we want an interrupt on completion, and reset the error counter
1386 pqTD->nStatus = 0x80;
1387 pqTD->bIoc = pQH->nEndpoint && pid == UsbPidIn;
1388 pqTD->nErr = 3; // Up to 3 retries of this transaction
1389
1390 // Set up the transfer
1391 pqTD->nBytes = nBytes;
1392 pqTD->nBufferSize = nBytes;
1393 pqTD->bDataToggle = bToggle;
1394
1395 if (nBytes) {
1396 // EHCI qTDs always describe five 4 KiB hardware pages, independently of
1397 // the target VM page size.
1398 const size_t nBufferPageOffset = pBuffer & EhciDescriptorOffsetMask;
1399 const uintptr_t pBufferPageStart = pBuffer & ~uintptr_t(EhciDescriptorOffsetMask);
1400 pqTD->nOffset = nBufferPageOffset;
1401
1402 const uintptr_t highestVirtual = static_cast<uintptr_t>(-1);
1403 if ((nBytes - 1) > highestVirtual - pBuffer) {
1404 ERROR("EHCI: addTransferToTransaction: buffer range wraps");
1405 pQH->pMetaData->bBuildFailed = true;
1406 m_qTDBitmap.clear(nIndex);
1407 ByteSet(pqTD, 0, sizeof(qTD));
1408 return;
1409 }
1410
1411 if (nBytes > EhciQtdBufferWindowBytes - nBufferPageOffset) {
1412 ERROR(
1413 "EHCI: addTransferToTransaction: Too many bytes for a single "
1414 "transaction!");
1415 pQH->pMetaData->bBuildFailed = true;
1416 m_qTDBitmap.clear(nIndex);
1417 ByteSet(pqTD, 0, sizeof(qTD));
1418 return;
1419 }
1420
1421 VirtualAddressSpace& va = Processor::information().getVirtualAddressSpace();
1422 uint32_t bufferPages[EhciQtdBufferPageCount] = {};
1423 const size_t bufferSpan = nBufferPageOffset + nBytes;
1424 bool mapped = true;
1425 bool dmaAddressable = true;
1426 for (size_t page = 0; page < EhciQtdBufferPageCount; ++page) {
1427 if (bufferSpan <= page * EhciHardwarePageBytes)
1428 break;
1429
1430 physical_uintptr_t physicalPage = 0;
1431 const uintptr_t virtualPage = pBufferPageStart + page * EhciHardwarePageBytes;
1432 if (!DriverDma::virtualToPhysical(va, virtualPage, physicalPage)) {
1433 mapped = false;
1434 continue;
1435 }
1436 if ((physicalPage & EhciDescriptorOffsetMask) ||
1437 !DriverDma::physicalRangeFits(physicalPage, EhciHardwarePageBytes,
1438 EhciHighestDmaAddress)) {
1439 dmaAddressable = false;
1440 } else {
1441 bufferPages[page] = physicalPage >> EhciHardwarePageShift;
1442 }
1443 }
1444
1445 if (!mapped) {
1446 ERROR("EHCI: addTransferToTransaction: buffer is not mapped");
1447 pQH->pMetaData->bBuildFailed = true;
1448 m_qTDBitmap.clear(nIndex);
1449 ByteSet(pqTD, 0, sizeof(qTD));
1450 return;
1451 }
1452
1453 if (!dmaAddressable) {
1454 auto* region = new MemoryRegion("EHCI transfer bounce");
1455 const size_t pages = DriverDma::pageCountForBytes(nBytes, EhciHardwarePageBytes);
1456 if (!region ||
1457 !PhysicalMemoryManager::instance().allocateRegion(
1458 *region, pages, PhysicalMemoryManager::continuous | PhysicalMemoryManager::below4GB,
1459 VirtualAddressSpace::KernelMode | VirtualAddressSpace::Write)) {
1460 delete region;
1461 ERROR("EHCI: addTransferToTransaction: couldn't allocate a below-4GiB bounce buffer");
1462 pQH->pMetaData->bBuildFailed = true;
1463 m_qTDBitmap.clear(nIndex);
1464 ByteSet(pqTD, 0, sizeof(qTD));
1465 return;
1466 }
1467
1468 auto* transfer = new EhciDmaTransfer{region, pBuffer, nBytes, pid == UsbPidIn};
1469 if (!transfer ||
1470 !DriverDma::physicalRangeFits(region->physicalAddress(), pages * EhciHardwarePageBytes,
1471 EhciHighestDmaAddress)) {
1472 delete transfer;
1473 delete region;
1474 ERROR("EHCI: addTransferToTransaction: bounce buffer is not DMA-addressable");
1475 pQH->pMetaData->bBuildFailed = true;
1476 m_qTDBitmap.clear(nIndex);
1477 ByteSet(pqTD, 0, sizeof(qTD));
1478 return;
1479 }
1480
1481 pqTD->pDma = reinterpret_cast<uintptr_t>(transfer);
1482 pqTD->nOffset = 0;
1483 if (pid != UsbPidIn)
1484 MemoryCopy(region->virtualAddress(), reinterpret_cast<void*>(pBuffer), nBytes);
1485 for (size_t page = 0; page < EhciQtdBufferPageCount; ++page) {
1486 if (nBytes <= page * EhciHardwarePageBytes)
1487 break;
1488 const physical_uintptr_t physicalPage =
1489 region->physicalAddress() + page * EhciHardwarePageBytes;
1490 bufferPages[page] = physicalPage >> EhciHardwarePageShift;
1491 }
1492 EMIT_IF(UsbVerboseDebug) {
1493 DEBUG_LOG("EHCI: using a below-4GiB bounce buffer for " << Dec << nBytes << " bytes");
1494 }
1495 }
1496
1497 pqTD->pPage0 = bufferPages[0];
1498 pqTD->pPage1 = bufferPages[1];
1499 pqTD->pPage2 = bufferPages[2];
1500 pqTD->pPage3 = bufferPages[3];
1501 pqTD->pPage4 = bufferPages[4];
1502 }
1503
1504 if (!pQH->nEndpoint && !nBytes && pQH->pMetaData->pFirstQTD) {
1505 qTD* previous = pQH->pMetaData->pFirstQTD;
1506 for (;;) {
1507 if (previous->nPid == 1) {
1508 previous->pAltNext = PHYS_QTD(nIndex) >> 5;
1509 previous->bAltNextInvalid = 0;
1510 }
1511 if (previous->bNextInvalid)
1512 break;
1513 previous = &m_pqTDList[((previous->pNext << 5) - m_pqTDListPhys) / sizeof(qTD)];
1514 }
1515 }
1516 // Add our qTD to the transaction.
1517 if (pQH->pMetaData->pLastQTD) {
1518 pQH->pMetaData->pLastQTD->pNext = PHYS_QTD(nIndex) >> 5;
1519 pQH->pMetaData->pLastQTD->bNextInvalid = 0;
1520
1521 if (pQH->pMetaData->pLastQTD == pQH->pMetaData->pFirstQTD) {
1522 pQH->overlay.pNext = pQH->pMetaData->pLastQTD->pNext;
1523 pQH->overlay.bNextInvalid = pQH->pMetaData->pLastQTD->bNextInvalid;
1524 }
1525 } else {
1526 pQH->pMetaData->pFirstQTD = pqTD;
1527 pQH->pQTD = PHYS_QTD(nIndex) >> 5;
1528 MemoryCopy(&pQH->overlay, pqTD, sizeof(qTD));
1529 }
1530 pQH->pMetaData->pLastQTD = pqTD;
1531}
1532
1533uintptr_t Ehci::createTransaction(UsbEndpoint endpointInfo) {
1534 OperationBarrier::Lease submission;
1535 if (!m_SubmissionOperations.tryAcquire(submission))
1536 return static_cast<uintptr_t>(-1);
1537 return createTransactionAdmitted(endpointInfo);
1538}
1539
1540uintptr_t Ehci::createTransactionAdmitted(UsbEndpoint endpointInfo) {
1541 LockGuard<Mutex> guard(m_Mutex);
1542 const size_t nIndex = m_QHBitmap.getFirstClear();
1543 if (nIndex >= (EhciQhRegionBytes / sizeof(QH))) {
1544 ERROR("USB: EHCI: QH space full");
1545 return static_cast<uintptr_t>(-1);
1546 }
1547 m_QHBitmap.set(nIndex);
1548
1549 QH* pQH = &m_pQHList[nIndex];
1550 ByteSet(pQH, 0, sizeof(QH));
1551
1552 // The pointer to the next QH gets set in doAsync
1553 pQH->nNextType = 1;
1554
1555 // NAK counter reload = 15
1556 pQH->nNakReload = 15;
1557
1558 // Head of the reclaim list
1559 pQH->hrcl = true;
1560
1561 // LS/FS handling
1562 pQH->nHubAddress = endpointInfo.speed != HighSpeed ? endpointInfo.nHubAddress : 0;
1563 pQH->nHubPort = endpointInfo.speed != HighSpeed ? endpointInfo.nHubPort : 0;
1564 pQH->bControlEndpoint = (endpointInfo.speed != HighSpeed) && !endpointInfo.nEndpoint;
1565
1566 // Data toggle controlled by qTD
1567 pQH->bDataToggleSrc = 1;
1568
1569 // Device address and speed
1570 pQH->nAddress = endpointInfo.nAddress;
1571 pQH->nSpeed = endpointInfo.speed == LowSpeed ? 1 : endpointInfo.speed == FullSpeed ? 0 : 2;
1572
1573 // Endpoint number and maximum packet size
1574 pQH->nEndpoint = endpointInfo.nEndpoint;
1575 pQH->nMaxPacketSize = endpointInfo.nMaxPacketSize;
1576
1577 // Bandwidth multiplier - number of transactions that can be performed in a
1578 // microframe
1579 pQH->mult = 1;
1580
1581 pQH->pMetaData = new QH::MetaData();
1582
1583 // Complete
1584 return nIndex;
1585}
1586
1587bool Ehci::doAsync(uintptr_t nTransaction, void (*pCallback)(uintptr_t, ssize_t),
1588 uintptr_t pParam) {
1589 OperationBarrier::Lease submission;
1590 if (!m_SubmissionOperations.tryAcquire(submission))
1591 return false;
1592
1593 LockGuard<Mutex> guard(m_Mutex);
1594 constexpr size_t QhCount = EhciQhRegionBytes / sizeof(QH);
1595 if ((nTransaction == static_cast<uintptr_t>(-1)) || nTransaction >= QhCount ||
1596 !m_QHBitmap.test(nTransaction)) {
1597 ERROR("EHCI: doAsync: didn't get a valid transaction id [" << nTransaction << "].");
1598 return false;
1599 }
1600
1601 QH* pQH = &m_pQHList[nTransaction];
1602 if (pQH->pMetaData &&
1603 pQH->pMetaData->completion.state() != UsbHcd::TransferCompletion::State::Idle) {
1604 ERROR("EHCI: doAsync: transaction is already submitted");
1605 return false;
1606 }
1607 if (!pQH->pMetaData || pQH->pMetaData->bBuildFailed || !pQH->pMetaData->pLastQTD ||
1608 pQH->pMetaData->bPeriodic) {
1609 ERROR("EHCI: doAsync: transaction could not be submitted [" << nTransaction << "].");
1610 reclaimQhLocked(nTransaction);
1611 return false;
1612 }
1613
1614 if (!m_pCurrentQueueTail || !m_pCurrentQueueHead || m_InterruptClosure >= 2) {
1615 ERROR("EHCI: asynchronous schedule is closed");
1616 reclaimQhLocked(nTransaction);
1617 return false;
1618 }
1619
1620 LockGuard<IrqProcessingLock> transactionGuard(m_IrqProcessingLock);
1621 pQH->pMetaData->pLastQTD->bIoc = 1;
1622
1623 // Only one transaction on this QH?
1624 if (pQH->pMetaData->pFirstQTD == pQH->pMetaData->pLastQTD) {
1625 // Update IOC bit in Transfer Overlay as well (as we have just changed)
1626 pQH->overlay.bIoc = 1;
1627 }
1628
1629 EMIT_IF(UsbVerboseDebug) {
1630 DEBUG_LOG("START #" << Dec << nTransaction << Hex << " " << Dec << pQH->nAddress << ":"
1631 << pQH->nEndpoint << Hex);
1632 }
1633
1634 // This QH is NOT the queue head. If we leave this set to one, and the
1635 // reclaim bit is set, the controller will think it's executed a full
1636 // circle, when in fact it's only partway there.
1637 pQH->hrcl = 0;
1638
1639 const size_t queueHeadIndex = m_pCurrentQueueHead - m_pQHList;
1640 pQH->pNext = (m_pQHListPhys + (queueHeadIndex * sizeof(QH))) >> 5;
1641 pQH->pMetaData->pNext = m_pCurrentQueueHead;
1642 pQH->pMetaData->pPrev = m_pCurrentQueueTail;
1643 QH* pOldTail = m_pCurrentQueueTail;
1644
1645 {
1646 LockGuard<Spinlock> queueGuard(m_QueueListChangeLock);
1647
1648 // Arming is the final software transition before the old tail makes the
1649 // QH visible to the controller. There are no fallible paths after it.
1650 pQH->pMetaData->completion.arm(pCallback, pParam, m_CompletionDeliveries.nextGeneration());
1651 m_pCurrentQueueTail = pQH;
1652 pOldTail->pNext = (m_pQHListPhys + (nTransaction * sizeof(QH))) >> 5;
1653 pOldTail->nNextType = 1;
1654 pOldTail->pMetaData->pNext = pQH;
1655 }
1656
1657 m_pCurrentQueueHead->hrcl = 1;
1658 return true;
1659}
1660
1661void Ehci::cancelAsyncAndDrain(uintptr_t nTransaction, void (*pCallback)(uintptr_t, ssize_t),
1662 uintptr_t pParam) {
1663 OperationBarrier::Lease cancellation;
1664 if (!m_CancelOperations.tryAcquire(cancellation))
1665 return;
1666
1667 List<UsbHcd::CallbackDeliveryQueue::Record*> completions;
1668 bool drainDelivery = false;
1669 UsbHcd::CallbackDeliveryQueue::Key deliveryKey = {0, 0};
1670
1671 {
1672 LockGuard<Mutex> guard(m_Mutex);
1673
1674 const bool teardownHalted = m_InterruptClosure >= 2;
1675 uint32_t savedInterrupts = 0;
1676 uint32_t savedCommand = 0;
1677 if (!teardownHalted && m_pBase && m_nOpRegsOffset) {
1678 savedInterrupts = m_pBase->read32(m_nOpRegsOffset + EHCI_INTR);
1679 savedCommand = m_pBase->read32(m_nOpRegsOffset + EHCI_CMD);
1680 m_pBase->write32(0, m_nOpRegsOffset + EHCI_INTR);
1681 m_pBase->write32(savedCommand & ~EHCI_CMD_RUN, m_nOpRegsOffset + EHCI_CMD);
1682 if (!waitForMmioState(m_pBase, m_nOpRegsOffset + EHCI_STS, EHCI_STS_HALTED,
1683 EHCI_STS_HALTED)) {
1684 panic(
1685 "EHCI cancellation could not establish the DMA halt "
1686 "boundary");
1687 }
1688 }
1689
1690 {
1691 LockGuard<IrqProcessingLock> irqGuard(m_IrqProcessingLock);
1692
1693 constexpr size_t QhCount = EhciQhRegionBytes / sizeof(QH);
1694 if ((nTransaction != static_cast<uintptr_t>(-1)) && nTransaction < QhCount &&
1695 m_QHBitmap.test(nTransaction)) {
1696 QH* pQH = &m_pQHList[nTransaction];
1697 QH::MetaData* metadata = pQH->pMetaData;
1698 if (metadata && !metadata->bPeriodic) {
1699 UsbHcd::TransferCompletion::Claim claim;
1700 const auto disposition =
1701 metadata->completion.claimCancellation(pCallback, pParam, -TransactionError, claim);
1702 if (disposition == UsbHcd::TransferCompletion::CancellationDisposition::Claimed) {
1703 if (!metadata->bIgnore) {
1704 LockGuard<Spinlock> queueGuard(m_QueueListChangeLock);
1705 QH* pPrev = metadata->pPrev;
1706 QH* pNext = metadata->pNext;
1707 if (pPrev && pNext) {
1708 if (pQH->hrcl)
1709 pNext->hrcl = 1;
1710 pPrev->pMetaData->pNext = pNext;
1711 pNext->pMetaData->pPrev = pPrev;
1712 pPrev->pNext = pQH->pNext;
1713 if (pQH == m_pCurrentQueueTail)
1714 m_pCurrentQueueTail = pPrev;
1715 }
1716 metadata->bIgnore = true;
1717 }
1718 deliveryKey = {nTransaction, claim.generation};
1719 captureCompletionLocked(nTransaction, pQH, claim, completions);
1720 } else if (disposition ==
1721 UsbHcd::TransferCompletion::CancellationDisposition::DrainPublished) {
1722 deliveryKey = {nTransaction, claim.generation};
1723 drainDelivery = true;
1724 assert(m_CompletionDeliveries.contains(deliveryKey));
1725 }
1726 }
1727 }
1728
1729 if (completions.count())
1730 m_CompletionDeliveries.publish(completions);
1731 }
1732
1733 if (!teardownHalted && m_pBase && m_nOpRegsOffset) {
1734 m_pBase->write32(savedCommand, m_nOpRegsOffset + EHCI_CMD);
1735 if (savedCommand & EHCI_CMD_RUN) {
1736 if (!waitForMmioState(m_pBase, m_nOpRegsOffset + EHCI_STS, EHCI_STS_HALTED, 0)) {
1737 panic(
1738 "EHCI cancellation could not establish the DMA "
1739 "restart boundary");
1740 }
1741 }
1742 {
1743 LockGuard<IrqProcessingLock> irqGuard(m_IrqProcessingLock);
1744 uint32_t restoredInterrupts = savedInterrupts;
1745 if (m_InterruptClosure == 1)
1746 restoredInterrupts &= ~EHCI_STS_PORTCH;
1747 m_pBase->write32(restoredInterrupts, m_nOpRegsOffset + EHCI_INTR);
1748 (void)m_pBase->read32(m_nOpRegsOffset + EHCI_INTR);
1749 }
1750 }
1751 }
1752
1753 if (completions.count()) {
1754 auto* completion = completions.popFront();
1755 (void)m_CompletionDeliveries.drain(deliveryKey);
1756 m_CompletionDeliveries.deliver(completion);
1757 } else if (drainDelivery)
1758 (void)m_CompletionDeliveries.drain(deliveryKey);
1759}
1760
1761bool Ehci::cancelInterruptInAndDrain(const UsbInterruptInToken& token,
1762 void (*callback)(uintptr_t, ssize_t), uintptr_t parameter,
1763 bool producerAlreadyStopped) {
1764 Machine::setShutdownDetail("EHCI: entering interrupt cancellation");
1765 OperationBarrier::Lease cancellation;
1766 if (!m_CancelOperations.tryAcquire(cancellation))
1767 panic("EHCI interrupt-IN cancellation raced controller teardown");
1768
1769 if (!producerAlreadyStopped) {
1770 bool matched = false;
1771 {
1772 Machine::setShutdownDetail("EHCI: locking controller for cancellation");
1773 LockGuard<Mutex> guard(m_Mutex);
1774 const bool teardownHalted = m_InterruptClosure >= 2;
1775 uint32_t savedInterrupts = 0;
1776 uint32_t savedCommand = 0;
1777 if (!teardownHalted && m_pBase && m_nOpRegsOffset) {
1778 Machine::setShutdownDetail("EHCI: halting interrupt DMA");
1779 savedInterrupts = m_pBase->read32(m_nOpRegsOffset + EHCI_INTR);
1780 savedCommand = m_pBase->read32(m_nOpRegsOffset + EHCI_CMD);
1781 m_pBase->write32(0, m_nOpRegsOffset + EHCI_INTR);
1782 m_pBase->write32(savedCommand & ~EHCI_CMD_RUN, m_nOpRegsOffset + EHCI_CMD);
1783 if (!waitForMmioState(m_pBase, m_nOpRegsOffset + EHCI_STS, EHCI_STS_HALTED,
1784 EHCI_STS_HALTED)) {
1785 panic("EHCI interrupt-IN cancellation could not halt DMA");
1786 }
1787 }
1788
1789 Machine::setShutdownDetail("EHCI: reclaiming interrupt descriptors");
1790 {
1791 LockGuard<IrqProcessingLock> irqGuard(m_IrqProcessingLock);
1792 constexpr size_t QhCount = EhciQhRegionBytes / sizeof(QH);
1793 if (token.transaction < QhCount && m_QHBitmap.test(token.transaction)) {
1794 QH* pQH = &m_pQHList[token.transaction];
1795 QH::MetaData* metadata = pQH->pMetaData;
1796 if (metadata && metadata->bPeriodic && metadata->periodicGeneration == token.generation &&
1797 metadata->pCallback == callback && metadata->pParam == parameter) {
1798 metadata->bIgnore = true;
1799 metadata->pCallback = nullptr;
1800 metadata->pParam = 0;
1801 reclaimQhLocked(token.transaction);
1802 rebuildPeriodicScheduleLocked();
1803 matched = true;
1804 }
1805 }
1806 }
1807
1808 if (!teardownHalted && m_pBase && m_nOpRegsOffset) {
1809 Machine::setShutdownDetail("EHCI: restarting controller after cancellation");
1810 m_pBase->write32(savedCommand, m_nOpRegsOffset + EHCI_CMD);
1811 Machine::setShutdownDetail("EHCI: waiting for controller to run");
1812 if ((savedCommand & EHCI_CMD_RUN) &&
1813 !waitForMmioState(m_pBase, m_nOpRegsOffset + EHCI_STS, EHCI_STS_HALTED, 0)) {
1814 panic("EHCI interrupt-IN cancellation could not restart DMA");
1815 }
1816 Machine::setShutdownDetail("EHCI: locking interrupt state after restart");
1817 LockGuard<IrqProcessingLock> irqGuard(m_IrqProcessingLock);
1818 uint32_t restoredInterrupts = savedInterrupts;
1819 if (m_InterruptClosure == 1)
1820 restoredInterrupts &= ~EHCI_STS_PORTCH;
1821 m_pBase->write32(restoredInterrupts, m_nOpRegsOffset + EHCI_INTR);
1822 (void)m_pBase->read32(m_nOpRegsOffset + EHCI_INTR);
1823 }
1824 }
1825
1826 if (!matched)
1827 panic("EHCI interrupt-IN handle lost its controller subscription");
1828 }
1829
1830#if X86_COMMON
1831 Machine::setShutdownDetail("EHCI: draining interrupt callbacks");
1832 return m_CompletionDeliveries.cancelSubscription(token.transaction, token.generation);
1833#else
1834 panic("non-x86 EHCI unexpectedly owned an interrupt-IN handle");
1835 return false;
1836#endif
1837}
1838
1839bool Ehci::addInterruptInHandler(UsbEndpoint endpointInfo, uintptr_t pBuffer, uint16_t nBytes,
1840 void (*pCallback)(uintptr_t, ssize_t),
1841 UsbInterruptInHandle& handle, uintptr_t pParam) {
1842#if !X86_COMMON
1843 (void)endpointInfo;
1844 (void)pBuffer;
1845 (void)nBytes;
1846 (void)pCallback;
1847 (void)handle;
1848 (void)pParam;
1849 ERROR("USB: non-x86 EHCI recurring interrupt-IN is not safely supported");
1850 return false;
1851#else
1852 OperationBarrier::Lease submission;
1853 if (!m_SubmissionOperations.tryAcquire(submission) || handle || !pCallback)
1854 return false;
1855
1856 if (!endpointInfo.nInterval || !nBytes || nBytes > endpointInfo.nMaxPacketSize ||
1857 (endpointInfo.speed == HighSpeed && endpointInfo.nInterval > 14) ||
1858 (endpointInfo.speed != HighSpeed && (!endpointInfo.nHubAddress || !endpointInfo.nHubPort)))
1859 return false;
1860 size_t microframes = endpointInfo.speed == HighSpeed ? size_t{1} << (endpointInfo.nInterval - 1)
1861 : size_t{8} * endpointInfo.nInterval;
1862 size_t interval = 1;
1863 while (interval < 1024 && interval * 16 <= microframes)
1864 interval *= 2;
1865 // Create a new transaction
1866 uintptr_t nTransaction = createTransactionAdmitted(endpointInfo);
1867 if (nTransaction == static_cast<uintptr_t>(-1)) {
1868 LockGuard<Mutex> guard(m_Mutex);
1869 return false;
1870 }
1871
1872 // Get the QH and set the periodic flag
1873 {
1874 LockGuard<Mutex> guard(m_Mutex);
1875 m_pQHList[nTransaction].pMetaData->bPeriodic = true;
1876 }
1877
1878 // Add a single transfer to the transaction
1879 addTransferToTransactionAdmitted(nTransaction, false, UsbPidIn, pBuffer, nBytes);
1880
1881 // Add the QH to the frame list
1882 {
1883 LockGuard<Mutex> guard(m_Mutex);
1884 QH* pQH = &m_pQHList[nTransaction];
1885 if (!pQH->pMetaData || pQH->pMetaData->bBuildFailed || !pQH->pMetaData->pLastQTD ||
1886 m_InterruptClosure >= 2) {
1887 ERROR("USB: EHCI: Couldn't add interrupt transfer!");
1888 reclaimQhLocked(nTransaction);
1889 return false;
1890 }
1891
1892 uint8_t startMask = 1, completeMask = 0;
1893 if (endpointInfo.speed == HighSpeed) {
1894 const size_t step = microframes < 8 ? microframes : 8;
1895 startMask = 0;
1896 for (size_t microframe = nTransaction % step; microframe < 8; microframe += step)
1897 startMask |= 1U << microframe;
1898 } else {
1899 uint8_t occupied = 0;
1900 for (size_t i = 1; i < EhciQhRegionBytes / sizeof(QH); ++i) {
1901 auto* other = &m_pQHList[i];
1902 if (i != nTransaction && m_QHBitmap.test(i) && other->pMetaData &&
1903 other->pMetaData->bPeriodic && other->pMetaData->pCallback &&
1904 other->nHubAddress == endpointInfo.nHubAddress)
1905 occupied |= other->ism;
1906 }
1907 size_t microframe = 0;
1908 while (microframe < 4 && (occupied & (1U << microframe)))
1909 ++microframe;
1910 if (microframe == 4) {
1911 reclaimQhLocked(nTransaction);
1912 return false;
1913 }
1914 startMask = 1U << microframe;
1915 completeMask = 0x1cU << microframe;
1916 }
1917 // All frame branches meet at frame zero. Bound the busiest microframe
1918 // conservatively, reserving 20 us for each split start/completion attempt.
1919 for (size_t microframe = 0; microframe < 8; ++microframe) {
1920 size_t nanoseconds = 0;
1921 for (size_t i = 1; i < EhciQhRegionBytes / sizeof(QH); ++i) {
1922 auto* other = &m_pQHList[i];
1923 if (!m_QHBitmap.test(i) || !other->pMetaData || !other->pMetaData->bPeriodic ||
1924 (i != nTransaction && !other->pMetaData->pCallback))
1925 continue;
1926 const uint8_t starts = i == nTransaction ? startMask : other->ism;
1927 const uint8_t completes = i == nTransaction ? completeMask : other->scm;
1928 if ((starts | completes) & (1U << microframe))
1929 nanoseconds += other->nSpeed == 2 ? other->nMaxPacketSize * 20U + 3000U : 20000U;
1930 }
1931 if (nanoseconds > 80000) {
1932 reclaimQhLocked(nTransaction);
1933 return false;
1934 }
1935 }
1936 const uint32_t savedCommand = m_pBase->read32(m_nOpRegsOffset + EHCI_CMD);
1937 m_pBase->write32(savedCommand & ~EHCI_CMD_PERIODICLE, m_nOpRegsOffset + EHCI_CMD);
1938 if (!waitForMmioState(m_pBase, m_nOpRegsOffset + EHCI_STS, 1U << 14, 0))
1939 panic("EHCI periodic insertion could not stop the periodic schedule");
1940 LockGuard<IrqProcessingLock> transactionGuard(m_IrqProcessingLock);
1941 pQH->hrcl = false;
1942 pQH->nNakReload = 0;
1943 pQH->ism = startMask;
1944 pQH->scm = completeMask;
1945 pQH->pMetaData->pLastQTD->nErr = 3;
1946 pQH->pMetaData->pLastQTD->bIoc = 1;
1947 pQH->pMetaData->periodicTemplate = *pQH->pMetaData->pLastQTD;
1948 MemoryCopy(&pQH->overlay, pQH->pMetaData->pLastQTD, sizeof(qTD));
1949 pQH->pMetaData->pCallback = pCallback;
1950 pQH->pMetaData->pParam = pParam;
1951 pQH->pMetaData->periodicGeneration = m_CompletionDeliveries.nextGeneration();
1952 pQH->pMetaData->periodicFrameIndex = 0;
1953 pQH->pMetaData->periodicInterval = interval;
1954 const bool published = publishInterruptInHandle(
1955 handle, {nTransaction, pQH->pMetaData->periodicGeneration}, pCallback, pParam);
1956 if (!published)
1957 panic("EHCI submitted interrupt-IN without publishing its owner handle");
1958 rebuildPeriodicScheduleLocked();
1959 m_pBase->write32(savedCommand | EHCI_CMD_PERIODICLE, m_nOpRegsOffset + EHCI_CMD);
1960 (void)m_pBase->read32(m_nOpRegsOffset + EHCI_CMD);
1961 }
1962 return true;
1963#endif
1964}
1965
1966void Ehci::replaySuppressedConnectionChange(size_t port) {
1967#if THREADS
1968 if (port >= m_nPorts) {
1969 ERROR("EHCI: invalid suppressed root-port replay " << Dec << port);
1970 return;
1971 }
1972
1973 LockGuard<IrqProcessingLock> irqGuard(m_IrqProcessingLock);
1974 // Teardown stops publication before its active enumeration worker returns.
1975 if (m_InterruptClosure) {
1976 return;
1977 }
1978 const auto observation = m_PortChanges[port].observe();
1979 const bool accepted = UsbHcd::PortChangeRequest::canAcknowledge(observation.result);
1980 if (accepted) {
1981 m_PortChanges[port].acknowledge(observation.generation);
1982 return;
1983 }
1984
1985 m_InterruptClosure = 1;
1986 const uint32_t interrupts = m_pBase->read32(m_nOpRegsOffset + EHCI_INTR);
1987 m_pBase->write32(interrupts & ~EHCI_STS_PORTCH, m_nOpRegsOffset + EHCI_INTR);
1988 (void)m_pBase->read32(m_nOpRegsOffset + EHCI_INTR);
1989 ERROR("EHCI: live suppressed root-port replay could not be published");
1990 assert(false);
1991#else
1992 (void)port;
1993#endif
1994}
1995
1996void Ehci::modifyPortControl(size_t portRegister, uint32_t clearMask, uint32_t setMask) {
1997 LockGuard<IrqProcessingLock> irqGuard(m_IrqProcessingLock);
1998 uint32_t control = portControlValue(m_pBase->read32(portRegister));
1999 control &= ~clearMask;
2000 control |= portControlValue(setMask);
2001 m_pBase->write32(control, portRegister);
2002 (void)m_pBase->read32(portRegister);
2003}
2004
2005bool Ehci::portReset(uint8_t nPort, bool bErrorResponse) {
2006 StartupActivity startup(this);
2007 if (!startup || !m_HardwareOwned || nPort >= m_nPorts)
2008 return false;
2009
2010 const size_t portRegister = m_nOpRegsOffset + EHCI_PORTSC + (nPort * 4);
2011 if (bErrorResponse) {
2012 modifyPortControl(portRegister, EHCI_PORTSC_EN, 0);
2013 if (!waitForMmioState(m_pBase, portRegister, EHCI_PORTSC_EN, 0)) {
2014 ERROR("EHCI: port " << Dec << nPort << Hex << " did not disable within 100 ms");
2015 return false;
2016 }
2017 }
2018
2019 int retry;
2020 for (retry = 0; retry < 3; retry++) {
2021 EMIT_IF(UsbVerboseDebug) {
2022 DEBUG_LOG("USB: EHCI: Port " << Dec << nPort << Hex << " - status before reset: "
2023 << m_pBase->read32(m_nOpRegsOffset + EHCI_PORTSC + (nPort * 4)));
2024 }
2025
2026 // Set the reset bit
2027 modifyPortControl(portRegister, 0, EHCI_PORTSC_PRES);
2028
2029 Time::delay(50 * Time::Multiplier::Millisecond);
2030
2031 // Unset the reset bit
2032 modifyPortControl(portRegister, EHCI_PORTSC_PRES, 0);
2033
2034 // Wait for the reset to complete
2035 if (!waitForMmioState(m_pBase, portRegister, EHCI_PORTSC_PRES, 0)) {
2036 ERROR("EHCI: reset on port " << Dec << nPort << Hex << " did not clear within 100 ms");
2037 return false;
2038 }
2039
2040 EMIT_IF(UsbVerboseDebug) {
2041 DEBUG_LOG("USB: EHCI: Port " << Dec << nPort << Hex << " - status after reset: "
2042 << m_pBase->read32(m_nOpRegsOffset + EHCI_PORTSC + (nPort * 4)));
2043 }
2044
2045 if ((m_pBase->read32(m_nOpRegsOffset + EHCI_PORTSC + (nPort * 4)) & EHCI_PORTSC_EN) &&
2046 (m_pBase->read32(m_nOpRegsOffset + EHCI_PORTSC + (nPort * 4)) & EHCI_PORTSC_CONN)) {
2047 DEBUG_LOG("USB: EHCI: Port " << Dec << nPort << Hex << " is connected");
2048 return true;
2049 } else {
2050 DEBUG_LOG("USB: EHCI: Port " << Dec << nPort << Hex
2051 << " seems to be not HighSpeed. Returning "
2052 "to companion controllers.");
2053 modifyPortControl(portRegister, 0, 0x2000);
2054 }
2055 }
2056
2057 WARNING("EHCI: Port " << Dec << nPort << Hex << " could not be connected");
2058
2059 return false;
2060}
2061
2062uint64_t Ehci::executeRequest(uint64_t p1, uint64_t p2, uint64_t p3, uint64_t p4, uint64_t p5,
2063 uint64_t p6, uint64_t p7, uint64_t p8) {
2064 if (p1 >= m_nPorts) {
2065 return 0;
2066 }
2067 UsbHcd::PortChangeRequest::Completion completion(m_PortChanges[p1], static_cast<size_t>(p8));
2068 if (!completion) {
2069 return 0;
2070 }
2071
2072 struct InitialPortCompletion {
2073 Ehci* controller;
2074 size_t port;
2075 ~InitialPortCompletion() {
2076 controller->completeInitialPort(port);
2077 }
2078 } initial{this, static_cast<size_t>(p1)};
2079 StartupActivity startup(this);
2080 if (!startup)
2081 return 0;
2082
2083 // See if there's any device attached on the port
2084 if (m_pBase->read32(m_nOpRegsOffset + EHCI_PORTSC + p1 * 4) & EHCI_PORTSC_CONN) {
2085 if (portReset(p1)) {
2086 if (!deviceConnected(p1, HighSpeed)) {
2087 WARNING("EHCI: Port " << Dec << p1 << Hex
2088 << " appeared to be connected but could not be set up");
2089 startup.failed();
2090 }
2091 } else {
2092 const uint32_t port = m_pBase->read32(m_nOpRegsOffset + EHCI_PORTSC + p1 * 4);
2093 // Companion-owned low/full-speed ports are not failed EHCI devices.
2094 if ((port & EHCI_PORTSC_CONN) && !(port & (1U << 13)))
2095 startup.failed();
2096 }
2097 } else {
2098 DEBUG_LOG("USB: EHCI: Port " << Dec << p1 << Hex << " is disconnected");
2099
2100 deviceDisconnected(p1);
2101 }
2102 return 0;
2103}
2104
2105void Ehci::cancelRequest(const Request& request) {
2106 if (request.p1 < m_nPorts) {
2107 m_PortChanges[request.p1].cancel(static_cast<size_t>(request.p8));
2108 completeInitialPort(request.p1);
2109 }
2110}
Device::Address::m_Io
IoBase * m_Io
Definition Device.h:115
Device::Address::m_Size
size_t m_Size
Definition Device.h:109
Device::Address::map
void map(size_t forcedSize=0, bool bUser=false, bool bWriteCombine=false, bool bWriteThrough=false)
Definition Device.cc:346
Device::addresses
virtual Vector< Address * > & addresses()
Definition Device.h:266
Device::getParent
Device * getParent() const
Definition Device.h:166
Device::getInterruptNumber
virtual uintptr_t getInterruptNumber()
Definition Device.h:271
Ehci
Definition Ehci.h:57
Ehci::m_CompletionDeliveries
UsbHcd::CallbackDeliveryQueue m_CompletionDeliveries
Definition Ehci.h:332
Ehci::cancelInterruptInAndDrain
MUST_USE_RESULT bool cancelInterruptInAndDrain(const UsbInterruptInToken &token, void(*callback)(uintptr_t, ssize_t), uintptr_t parameter, bool producerAlreadyStopped) override
Definition Ehci.cc:1761
Ehci::replaySuppressedConnectionChange
void replaySuppressedConnectionChange(size_t port) override
Definition Ehci.cc:1966
Ehci::irq
IrqDisposition irq(irq_id_t number) override
IRQ handler.
Definition Ehci.cc:1012
Ehci::portControlValue
static uint32_t portControlValue(uint32_t value)
Definition Ehci.h:280
Ehci::m_DequeueCount
Semaphore m_DequeueCount
Definition Ehci.h:358
Ehci::addTransferToTransaction
virtual void addTransferToTransaction(uintptr_t pTransaction, bool bToggle, UsbPid pid, uintptr_t pBuffer, size_t nBytes)
Adds a new transfer to an existent transaction.
Definition Ehci.cc:1313
Ehci::portReset
virtual bool portReset(uint8_t nPort, bool bErrorResponse=false)
Gets a UsbDevice from a given vendor:product pair.
Definition Ehci.cc:2005
Ehci::doAsync
virtual MUST_USE_RESULT bool doAsync(uintptr_t pTransaction, void(*pCallback)(uintptr_t, ssize_t)=0, uintptr_t pParam=0)
Definition Ehci.cc:1587
Ehci::m_IrqProcessingLock
IrqProcessingLock m_IrqProcessingLock
Definition Ehci.h:330
Ehci::m_InterruptClosure
Atomic< size_t > m_InterruptClosure
Definition Ehci.h:371
Ehci::m_SubmissionOperations
OperationBarrier m_SubmissionOperations
Definition Ehci.h:363
Ehci::cancelAsyncAndDrain
virtual void cancelAsyncAndDrain(uintptr_t pTransaction, void(*pCallback)(uintptr_t, ssize_t), uintptr_t pParam)
Definition Ehci.cc:1661
Ehci::executeRequest
virtual uint64_t executeRequest(uint64_t p1=0, uint64_t p2=0, uint64_t p3=0, uint64_t p4=0, uint64_t p5=0, uint64_t p6=0, uint64_t p7=0, uint64_t p8=0)
Definition Ehci.cc:2062
Ehci::m_CallbackOperations
OperationBarrier m_CallbackOperations
Definition Ehci.h:367
Ehci::m_CancelOperations
OperationBarrier m_CancelOperations
Definition Ehci.h:365
Ehci::addInterruptInHandler
virtual MUST_USE_RESULT bool addInterruptInHandler(UsbEndpoint endpointInfo, uintptr_t pBuffer, uint16_t nBytes, void(*pCallback)(uintptr_t, ssize_t), UsbInterruptInHandle &handle, uintptr_t pParam=0)
Adds an owned recurring interrupt-IN transaction.
Definition Ehci.cc:1839
Ehci::cancelRequest
void cancelRequest(const Request &request) override
Definition Ehci.cc:2105
Ehci::createTransaction
virtual uintptr_t createTransaction(UsbEndpoint endpointInfo)
Creates a new transaction with the given endpoint data.
Definition Ehci.cc:1533
ExtensibleBitmap::test
bool test(size_t n) const
Definition ExtensibleBitmap.cc:177
ExtensibleBitmap::clear
void clear(size_t n)
Definition ExtensibleBitmap.cc:138
ExtensibleBitmap::set
void set(size_t n)
Definition ExtensibleBitmap.cc:81
IoBase
Abstrace base class for hardware I/O capabilities.
Definition IoBase.h:32
IoBase::write32
virtual void write32(uint32_t value, size_t offset=0)=0
IoBase::read32
virtual uint32_t read32(size_t offset=0)=0
IoBase::size
virtual size_t size() const =0
IrqManager::registerPciIrqHandler
virtual irq_id_t registerPciIrqHandler(IrqHandler *handler, Device *pDevice, const IrqPolicy &policy)=0
IrqManager::control
virtual bool control(uint8_t irq, ControlCode code, size_t argument)
Definition IrqManager.cc:55
List
Definition List.h:61
Machine::setShutdownDetail
static void setShutdownDetail(const char *detail)
Definition Machine.cc:38
MemoryRegion
Special memory entity in the kernel's virtual address space.
Definition MemoryRegion.h:35
MemoryRegion::virtualAddress
void * virtualAddress() const
Definition MemoryRegion.cc:41
MemoryRegion::physicalAddress
physical_uintptr_t physicalAddress() const
Definition MemoryRegion.cc:45
OperationBarrier::tryAcquire
MUST_USE_RESULT bool tryAcquire(Lease &lease)
Definition OperationBarrier.cc:67
OwnedThread::join
void join()
Definition OwnedThread.h:49
PhysicalMemoryManager::instance
static PhysicalMemoryManager & instance()
Definition arm64/PhysicalMemoryManager.cc:34
ProcessorBase::information
static ProcessorInformation & information()
Definition x64/Processor.h:30
ProcessorBase::pause
static void pause()
Definition arm64/Processor.cc:180
RequestQueue::getLifecycleState
LifecycleState getLifecycleState()
Definition RequestQueue.cc:847
RequestQueue::destroy
virtual void destroy()
Definition RequestQueue.cc:434
RequestQueue::initialise
virtual void initialise()
Definition RequestQueue.cc:261
Semaphore::release
void release(size_t n=1)
Definition Semaphore.cc:549
Semaphore::acquireForCompletion
MUST_USE_RESULT bool acquireForCompletion(size_t n=1, size_t timeoutSecs=0, size_t timeoutUsecs=0)
Definition Semaphore.cc:372
TargetInfo::getPageSize
static constexpr size_t getPageSize() noexcept
Definition TargetInfo.h:40
UsbHcd::CallbackDeliveryQueue::cancelSubscription
bool cancelSubscription(uintptr_t transaction, size_t subscription)
Definition CallbackDelivery.h:221
UsbHcd::CallbackDeliveryQueue::publish
void publish(List< Record * > &records)
Definition CallbackDelivery.h:147
UsbHcd::TransferCompletion::claimCancellation
MUST_USE_RESULT CancellationDisposition claimCancellation(Callback callback, uintptr_t parameter, ssize_t cancellationResult, Claim &claim)
Definition TransferCompletion.h:98
UsbHcd::TransferCompletion::arm
void arm(Callback callback, uintptr_t parameter, size_t generation)
Definition TransferCompletion.h:67
UsbHcd::TransferCompletion::captureNatural
MUST_USE_RESULT bool captureNatural(ssize_t result)
Definition TransferCompletion.h:77
UsbHub::deviceConnected
bool deviceConnected(uint8_t nPort, UsbSpeed speed)
Called when a device is connected to a port on the hub.
Definition UsbHub.cc:388
UsbHub::deviceDisconnected
void deviceDisconnected(uint8_t nPort)
Called when a device is disconnected from a port on the hub.
Definition UsbHub.cc:550
UsbHub::publishInterruptInHandle
MUST_USE_RESULT bool publishInterruptInHandle(UsbInterruptInHandle &handle, const UsbInterruptInToken &token, void(*callback)(uintptr_t, ssize_t), uintptr_t parameter)
Definition UsbHub.cc:207
UsbHub::disconnectAllDevices
void disconnectAllDevices()
Definition UsbHub.cc:217
EMIT_IF
#define EMIT_IF(...)
Definition system/include/pedigree/kernel/compiler.h:89
panic
void EXPORTED_PUBLIC panic(const char *msg) NORETURN
Definition panic.cc:118
Dec
@ Dec
Definition Log.h:126
Hex
@ Hex
Definition Log.h:124
IrqDisposition
IrqDisposition
Definition IrqHandler.h:31
IrqDisposition::Quiesced
@ Quiesced
List::popFront
T popFront()
Definition List.h:330
List::count
size_t count() const
Definition List.h:212
List::pushBack
void pushBack(const T &value)
Definition List.h:216
Ehci::QH::MetaData::pCallback
void(* pCallback)(uintptr_t, ssize_t)
Definition Ehci.h:172
Ehci::QH::MetaData::completion
UsbHcd::TransferCompletion completion
Definition Ehci.h:191
Generated on Sat Oct 3 2026 06:38:05 for The Pedigree Project by doxygen 1.9.8