The Pedigree Project 0.1
AhciPort.cc
1/*
2 * Copyright (c) 2026, Pedigree Developers
3 * SPDX-License-Identifier: ISC
4 *
5 * Permission to use, copy, modify, and distribute this software for any
6 * purpose with or without fee is hereby granted, provided that the above
7 * copyright notice and this permission notice appear in all copies.
8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15 */
16#include "AhciPort.h"
17#include "pedigree/kernel/LockGuard.h"
18#include "pedigree/kernel/Log.h"
19#include "pedigree/kernel/TargetInfo.h"
20#include "pedigree/kernel/machine/Machine.h"
21#include "pedigree/kernel/machine/Pci.h"
22#include "pedigree/kernel/panic.h"
23#include "pedigree/kernel/process/Scheduler.h"
24#include "pedigree/kernel/process/TerminationDeferral.h"
25#include "pedigree/kernel/process/Thread.h"
26#include "pedigree/kernel/processor/IoBase.h"
27#include "pedigree/kernel/processor/PhysicalMemoryManager.h"
28#include "pedigree/kernel/processor/Processor.h"
29#include "pedigree/kernel/processor/ProcessorInformation.h"
30#include "pedigree/kernel/processor/VirtualAddressSpace.h"
31#include "pedigree/kernel/time/Time.h"
32#include "pedigree/kernel/utilities/StaticString.h"
33#include "pedigree/kernel/utilities/utility.h"
34
35#include "Registers.h"
36#include "modules/drivers/common/DmaBuffer.h"
37
38using namespace Ahci;
39
40namespace {
41physical_uintptr_t usableDirectWrite(physical_uintptr_t physical, size_t bytes) {
42 const size_t pageSize = TargetInfo::getPageSize();
43 if (!physical || !bytes || bytes > pageSize || physical % pageSize ||
44 !DriverDma::physicalRangeFits(physical, bytes, 0xffffffffU)) {
45 return 0;
46 }
47 return physical;
48}
49} // namespace
50
51AhciPort::AhciPort(IoBase* registers, size_t port, Device* dmaDevice)
52 : m_Registers(registers),
53 m_DmaDevice(dmaDevice),
54 m_Port(port),
55 m_Control("AHCI command storage"),
56 m_Online(false),
57 m_WritesPending(true),
58 m_Active(0),
59 m_Queued(0),
60 m_SlotCount(1),
61 m_QueueDepth(0),
62 m_SectorBytes(512),
63 m_SupportsNcq(false),
64 m_AddressesInstalled(false),
65 m_PolledInterrupt(false),
66 m_InterruptCompletions(0),
67 m_MaximumOutstanding(0),
68 m_Outstanding(0) {}
69AhciPort::~AhciPort() {
70 shutdown();
71}
72uint32_t AhciPort::read(size_t reg) const {
73 return m_Registers->read32(PortBase + m_Port * PortStride + reg);
74}
75void AhciPort::write(size_t reg, uint32_t value) {
76 m_Registers->write32(value, PortBase + m_Port * PortStride + reg);
77}
78void AhciPort::waitForProgress() {
79 Thread* thread = Processor::information().getCurrentThread();
80 if (!thread || !Processor::getInterrupts()) {
81 Processor::pause();
82 } else if (thread->eventsDeferred()) {
83 // Mmap faults suppress timeout events along with other event callbacks.
84 Scheduler::instance().yield();
85 } else {
86 Time::delay(Time::Multiplier::Millisecond);
87 }
88}
89bool AhciPort::waitClear(size_t reg, uint32_t bits, size_t milliseconds) {
90 const auto deadline = Time::getTicks() + milliseconds * Time::Multiplier::Millisecond;
91 do {
92 if (!(read(reg) & bits))
93 return true;
94 waitForProgress();
95 } while (Time::getTicks() < deadline);
96 return !(read(reg) & bits);
97}
98bool AhciPort::stopEngines() {
99 // AHCI 10.3: FRE must remain set until the command-list engine has stopped.
100 write(Cmd, read(Cmd) & ~Start);
101 if (!waitClear(Cmd, CommandRunning, 500))
102 return false;
103 write(Cmd, read(Cmd) & ~FisEnable);
104 return waitClear(Cmd, FisRunning, 500);
105}
106void AhciPort::acknowledge(uint32_t status) {
107 // Some port status bits are backed by SError diagnostic bits.
108 const uint32_t error = read(Serr);
109 if (error)
110 write(Serr, error);
111 if (status)
112 write(PortIs, status);
113}
114bool AhciPort::initialise(uint32_t capabilities, uint32_t version, uint32_t extendedCapabilities) {
115 write(PortIe, 0);
116 if (!stopEngines()) {
117 ERROR("AHCI: port " << m_Port << " firmware engines did not stop");
118 return false;
119 }
120 const size_t pageSize = TargetInfo::getPageSize();
121 m_SlotCount = ((capabilities >> 8) & 31U) + 1;
122 m_SupportsNcq = capabilities & (1U << 30);
123 const size_t controlBytes = TableOffset + m_SlotCount * TableStride;
124 if (pageSize < 4096 || pageSize > MaxTransfer || (MaxTransfer % pageSize))
125 return false;
126 auto& memory = PhysicalMemoryManager::instance();
127 const size_t constraints = PhysicalMemoryManager::continuous | PhysicalMemoryManager::below4GB;
128 const size_t flags = VirtualAddressSpace::KernelMode | VirtualAddressSpace::Write;
129 if (!memory.allocateRegion(m_Control, (controlBytes + pageSize - 1) / pageSize, constraints,
130 flags))
131 return false;
132 ByteSet(m_Control.virtualAddress(), 0, m_Control.size());
133 for (size_t i = 0; i < m_SlotCount; ++i) {
134 // Contiguous allocations currently consume the scarce ISA DMA pool.
135 // PRDs allow each bounce page to come from ordinary memory below 4 GiB.
136 if (!memory.allocateRegion(m_Slots[i].data, MaxTransfer / pageSize,
137 PhysicalMemoryManager::below4GB, flags))
138 return false;
139 ByteSet(m_Slots[i].data.virtualAddress(), 0, m_Slots[i].data.size());
140 for (size_t page = 0; page < MaxTransfer / pageSize; ++page) {
141 size_t mappingFlags = 0;
142 VirtualAddressSpace::getKernelAddressSpace().getMapping(
143 static_cast<uint8_t*>(m_Slots[i].data.virtualAddress()) + page * pageSize,
144 m_Slots[i].pages[page], mappingFlags);
145 if (m_Slots[i].pages[page] >= (uint64_t{1} << 32))
146 return false;
147 }
148 }
149 FENCE();
150 write(Clb, static_cast<uint32_t>(m_Control.physicalAddress()));
151 write(Clbu, 0);
152 write(Fb, static_cast<uint32_t>(m_Control.physicalAddress() + FisOffset));
153 write(Fbu, 0);
154 m_AddressesInstalled = true;
155
156 uint32_t cmd = read(Cmd) & ~(Atapi | AggressivePower | IccMask);
157 if (cmd & ColdPresence)
158 cmd |= PowerOn;
159 if (capabilities & StaggeredSpinup)
160 cmd |= Spinup;
161 // Disable automatic device sleep only when its register is implemented.
162 if (version >= 0x00010300 && (extendedCapabilities & (1U << 3)))
163 write(Devslp, read(Devslp) & ~1U);
164 write(Cmd, cmd | FisEnable);
165 uint32_t control = read(Sctl) & 0xf0U; // Retain firmware's speed restriction.
166 control |= (version >= 0x00010300 ? 7U : 3U) << 8;
167 write(Serr, 0xffffffffU);
168 write(Sctl, control | 1U);
169 const auto resetUntil = Time::getTicks() + Time::Multiplier::Millisecond;
170 while (Time::getTicks() < resetUntil)
171 waitForProgress();
172 write(Sctl, control);
173 const auto linkDeadline = Time::getTicks() + Time::Multiplier::Second;
174 while ((read(Ssts) & 15U) != 3U && Time::getTicks() < linkDeadline)
175 waitForProgress();
176 write(Serr, 0xffffffffU);
177 if ((read(Ssts) & 15U) != 3U)
178 return false;
179 write(Cmd, (read(Cmd) & ~IccMask) | IccActive);
180 if (!waitClear(Tfd, Busy | DataRequest, 30000)) {
181 WARNING("AHCI: port " << m_Port << " device not ready after COMRESET");
182 return false;
183 }
184 if (read(Sig) != SataDisk) {
185 NOTICE("AHCI: port " << m_Port << " unsupported signature " << Hex << read(Sig));
186 return false;
187 }
188 if ((read(Cmd) & CommandRunning) || read(Ci) || read(Sact))
189 return false;
190 acknowledge(read(PortIs));
191 write(Cmd, read(Cmd) | Start);
192 (void)read(Cmd);
193 m_Online = true;
194 return true;
195}
196void AhciPort::enableInterrupts() {
197 LockGuard<Mutex> state(m_StateLock);
198 acknowledge(read(PortIs));
199 if (m_Online)
200 write(PortIe, PortInterrupts);
201}
202void AhciPort::configureDisk(size_t sectorBytes, size_t queueDepth) {
203 m_SectorBytes = sectorBytes;
204 m_QueueDepth = m_SupportsNcq ? (queueDepth < m_SlotCount ? queueDepth : m_SlotCount) : 0;
205 NOTICE("AHCI: port " << m_Port << " NCQ depth " << Dec << m_QueueDepth << Hex);
206}
207void AhciPort::observe(uint32_t status, bool fromInterrupt) {
208 if (!m_Active)
209 return;
210 uint32_t errors = status & PortErrors;
211 if (read(Tfd) & (TaskError | DeviceFault))
212 errors |= TaskFileError;
213 if ((read(Ssts) & 15U) != 3U)
214 errors |= 1U << 22;
215 // Without READ LOG EXT attribution, no outstanding tag can be trusted after
216 // an NCQ error. Stop admission before any slot is retired or reused.
217 if (errors) {
218 m_Online = false;
219 write(PortIe, 0);
220 }
221 const uint32_t pending = (read(Sact) & m_Queued) | (read(Ci) & ~m_Queued);
222 for (size_t i = 0; i < m_SlotCount; ++i) {
223 if (!(m_Active & (1U << i)))
224 continue;
225 Slot& slot = m_Slots[i];
226 slot.errors |= errors;
227 if (!slot.done && (slot.errors || !(pending & (1U << i)))) {
228 slot.done = true;
229 --m_Outstanding;
230 if (fromInterrupt)
231 ++m_InterruptCompletions;
232 slot.completion.release();
233 }
234 }
235}
236void AhciPort::pollCompletions(bool interrupts) {
237 const uint32_t status = read(PortIs);
238 // Polling can withdraw INTx before its already-dispatched IRQ worker runs.
239 // Capture enabled causes before observe() can disable a failed port's IRQs.
240 if (interrupts && (status & read(PortIe)))
241 m_PolledInterrupt = true;
242 observe(status, false);
243 acknowledge(status);
244}
245bool AhciPort::interrupt(bool pending) {
246 LockGuard<Mutex> state(m_StateLock);
247 const bool credited = m_PolledInterrupt;
248 m_PolledInterrupt = false;
249 const uint32_t status = pending ? read(PortIs) : 0;
250 if (!status)
251 return credited;
252 observe(status, true);
253 acknowledge(status);
254 return true;
255}
256bool AhciPort::chooseSlot(bool queued, size_t& index) {
257 LockGuard<Mutex> state(m_StateLock);
258 index = 32;
259 if (!m_Online)
260 return false;
261 if (queued || !m_Active) {
262 const size_t count = queued ? m_QueueDepth : 1;
263 for (size_t i = 0; i < count; ++i) {
264 if (!(m_Active & (1U << i))) {
265 index = i;
266 break;
267 }
268 }
269 }
270 return true;
271}
272
273bool AhciPort::issueCommand(size_t index, uint8_t opcode, uint64_t lba, uint16_t sectors,
274 void* buffer, size_t bytes, bool writing, bool queued, bool interrupts,
275 physical_uintptr_t directPhysical,
276 const physical_uintptr_t* directReadPages) {
277 Slot& slot = m_Slots[index];
278 const uint32_t mask = 1U << index;
279 auto* header = static_cast<CommandHeader*>(m_Control.virtualAddress()) + index;
280 const size_t tableOffset = TableOffset + index * TableStride;
281 auto* table = reinterpret_cast<CommandTable*>(
282 reinterpret_cast<uintptr_t>(m_Control.virtualAddress()) + tableOffset);
283 {
284 LockGuard<Mutex> state(m_StateLock);
285 if (!m_Online || (!queued && (read(Ci) || read(Sact) || (read(Tfd) & (Busy | DataRequest)))))
286 return false;
287 ByteSet(header, 0, sizeof(*header));
288 ByteSet(table, 0, sizeof(*table));
289 if (writing && bytes && !directPhysical) {
290 MemoryCopy(slot.data.virtualAddress(), buffer, bytes);
291 }
292 const size_t pageSize = TargetInfo::getPageSize();
293 const size_t prds = directPhysical ? 1 : (bytes + pageSize - 1) / pageSize;
294 header->flags = 5U | (writing ? 1U << 6 : 0U) | (prds << 16);
295 header->table = static_cast<uint32_t>(m_Control.physicalAddress() + tableOffset);
296 table->fis[0] = 0x27;
297 table->fis[1] = 0x80;
298 table->fis[2] = opcode;
299 if (queued || opcode == 0x25 || opcode == 0x35)
300 table->fis[7] = 0x40;
301 for (size_t i = 0; i < 3; ++i) {
302 table->fis[4 + i] = static_cast<uint8_t>(lba >> (i * 8));
303 table->fis[8 + i] = static_cast<uint8_t>(lba >> ((i + 3) * 8));
304 }
305 if (queued) {
306 table->fis[3] = static_cast<uint8_t>(sectors);
307 table->fis[11] = static_cast<uint8_t>(sectors >> 8);
308 table->fis[12] = static_cast<uint8_t>(index << 3);
309 } else {
310 table->fis[12] = static_cast<uint8_t>(sectors);
311 table->fis[13] = static_cast<uint8_t>(sectors >> 8);
312 }
313 for (size_t page = 0; page < prds; ++page) {
314 const size_t remaining = bytes - page * pageSize;
315 const size_t count = remaining < pageSize ? remaining : pageSize;
316 physical_uintptr_t dataAddress = slot.pages[page];
317 if (directReadPages) {
318 dataAddress = directReadPages[page];
319 } else if (directPhysical) {
320 dataAddress = directPhysical;
321 }
322 table->data[page].address = static_cast<uint32_t>(dataAddress);
323 table->data[page].byteCount = static_cast<uint32_t>(count - 1);
324 }
325 // Observe previous commands before acknowledging shared port status.
326 pollCompletions(interrupts);
327 if (!m_Online)
328 return false;
329 [[maybe_unused]] const size_t drained = slot.completion.drainAvailable();
330 slot.errors = 0;
331 slot.done = false;
332 m_Active |= mask;
333 ++m_Outstanding;
334 if (m_Outstanding > m_MaximumOutstanding)
335 m_MaximumOutstanding = m_Outstanding;
336 if (queued)
337 m_Queued |= mask;
338 FENCE();
339 if (queued)
340 write(Sact, mask);
341 const size_t timeoutSeconds = (opcode == 0xe7 || opcode == 0xea) ? 120 : 30;
342 slot.deadline = Time::getTicks() + timeoutSeconds * Time::Multiplier::Second;
343 write(Ci, mask);
344 (void)read(Ci);
345 }
346 return true;
347}
348
349bool AhciPort::reapCommand(size_t index, uint8_t opcode, void* buffer, size_t bytes, bool writing,
350 bool queued, bool interrupts, bool interruptProbe, bool directRead) {
351 Slot& slot = m_Slots[index];
352 const uint32_t mask = 1U << index;
353 auto* header = static_cast<CommandHeader*>(m_Control.virtualAddress()) + index;
354 bool success = false;
355 bool interruptGrace = interruptProbe;
356 for (;;) {
357 {
358 LockGuard<Mutex> state(m_StateLock);
359 // The readiness probe must observe a real IRQ before polling can
360 // consume its completion; ordinary owners check hardware before sleeping.
361 if (!slot.done && !interruptGrace) {
362 pollCompletions(interrupts);
363 }
364 if (slot.done || Time::getTicks() >= slot.deadline) {
365 FENCE();
366 // AHCI 5.4.1: PRDBC is not defined for native queued commands.
367 success = slot.done && !slot.errors && m_Online && !(read(queued ? Sact : Ci) & mask) &&
368 (queued || header->transferred == bytes);
369 if (!success) {
370 ERROR("AHCI: port " << m_Port << " command " << Hex << opcode << " tag " << index
371 << " failed, CI=" << read(Ci) << " SACT=" << read(Sact)
372 << " TFD=" << read(Tfd) << " errors=" << slot.errors);
373 m_Online = false;
374 write(PortIe, 0);
375 // Mark all owners failed before stopping engines clears hardware bits.
376 for (size_t i = 0; i < m_SlotCount; ++i) {
377 if (m_Active & (1U << i)) {
378 m_Slots[i].errors |= TaskFileError;
379 if (!m_Slots[i].done) {
380 m_Slots[i].done = true;
381 --m_Outstanding;
382 m_Slots[i].completion.release();
383 }
384 }
385 }
386 if (!stopEngines())
387 panic("AHCI: cannot stop failed port DMA; refusing to release memory");
388 } else if (!writing && bytes && !directRead) {
389 MemoryCopy(buffer, slot.data.virtualAddress(), bytes);
390 }
391 m_Active &= ~mask;
392 m_Queued &= ~mask;
393 break;
394 }
395 }
396 Thread* thread = Processor::information().getCurrentThread();
397 if (interrupts && thread && Processor::getInterrupts() && !thread->eventsDeferred()) {
398 const bool acquired = slot.completion.acquireForCompletion(1, interruptGrace ? 1 : 0,
399 interruptGrace ? 0 : 10000);
400 (void)acquired;
401 } else {
402 waitForProgress();
403 }
404 interruptGrace = false;
405 }
406 return success;
407}
408
409bool AhciPort::command(uint8_t opcode, uint64_t lba, uint16_t sectors, void* buffer, size_t bytes,
410 bool writing, bool interrupts, bool interruptProbe, bool cacheFill,
411 physical_uintptr_t dmaPhysical, const physical_uintptr_t* directReadPages) {
412 if (bytes > MaxTransfer || (bytes && (!buffer || (bytes & 1U))) || (lba >> 48))
413 return false;
414 TerminationDeferral lifetime;
415 physical_uintptr_t directPhysical =
416 writing && opcode == 0x35 ? usableDirectWrite(dmaPhysical, bytes) : 0;
417 PciBus::DmaMapping directReadMapping;
418 const size_t pageSize = TargetInfo::getPageSize();
419 // Ordinary eviction cannot retire an Editing cache page during this synchronous command.
420 if (!directReadPages && cacheFill && opcode == 0x25 && !writing && bytes && bytes <= pageSize &&
421 !(reinterpret_cast<uintptr_t>(buffer) & (pageSize - 1))) {
422 physical_uintptr_t physical = 0;
423 size_t flags = 0;
424 auto& addressSpace = VirtualAddressSpace::getKernelAddressSpace();
425 if (addressSpace.getMapping(buffer, physical, flags) && physical &&
426 !(physical & (pageSize - 1)) &&
427 (flags & (VirtualAddressSpace::KernelMode | VirtualAddressSpace::Write)) ==
428 (VirtualAddressSpace::KernelMode | VirtualAddressSpace::Write) &&
429 !(flags & (VirtualAddressSpace::Swapped | VirtualAddressSpace::CopyOnWrite |
430 VirtualAddressSpace::NoAccess | VirtualAddressSpace::Borrowed))) {
431 if (DriverDma::physicalRangeFits(physical, bytes, 0xffffffffU)) {
432 directPhysical = physical;
433 } else if (m_DmaDevice &&
434 PciBus::instance().mapDmaPage(m_DmaDevice, physical, bytes, directReadMapping)) {
435 directPhysical = directReadMapping.address();
436 }
437 }
438 }
439 LockGuard<Mutex> command(m_CommandLock);
440 const bool queued = m_QueueDepth && (opcode == 0x25 || opcode == 0x35);
441 if (queued)
442 opcode = writing ? 0x61 : 0x60;
443 const auto admissionDeadline = Time::getTicks() + 120 * Time::Multiplier::Second;
444 size_t index = 32;
445 while (true) {
446 if (!chooseSlot(queued, index))
447 return false;
448 if (index != 32)
449 break;
450 if (Time::getTicks() >= admissionDeadline)
451 return false;
452 waitForProgress();
453 }
454 const bool flush = (opcode == 0xe7 || opcode == 0xea) && !bytes && !writing;
455 if (flush && !m_WritesPending) {
456 return true;
457 }
458 if (writing) {
459 m_WritesPending = true;
460 }
461 if (!issueCommand(index, opcode, lba, sectors, buffer, bytes, writing, queued, interrupts,
462 directPhysical, directReadPages))
463 return false;
464 if (queued) {
465 m_CommandLock.release();
466 command.disown();
467 }
468 const bool succeeded =
469 reapCommand(index, opcode, buffer, bytes, writing, queued, interrupts, interruptProbe,
470 directPhysical != 0 || directReadPages != nullptr);
471 // A nonqueued barrier holds the submission gate through completion, so no
472 // later write can be mistaken for part of this successful flush.
473 if (flush && succeeded) {
474 m_WritesPending = false;
475 }
476 return succeeded;
477}
478
479bool AhciPort::readBatch(Disk::ReadBuffer* buffers, size_t count, bool interrupts) {
480 if (count > Disk::MaxReadBuffers || (count && !buffers)) {
481 return false;
482 }
483 for (size_t i = 0; i < count; ++i) {
484 buffers[i].complete = false;
485 }
486 const size_t pageSize = TargetInfo::getPageSize();
487 for (size_t i = 0; i < count; ++i) {
488 if (!buffers[i].length || buffers[i].length > pageSize) {
489 return false;
490 }
491 }
492 PciBus::DmaMapping mappings[Disk::MaxReadBuffers];
493 physical_uintptr_t dmaAddresses[Disk::MaxReadBuffers] = {};
494 auto& addressSpace = VirtualAddressSpace::getKernelAddressSpace();
495 for (size_t i = 0; i < count; ++i) {
496 const uintptr_t address = reinterpret_cast<uintptr_t>(buffers[i].buffer);
497 const size_t offset = address & (pageSize - 1);
498 if (!m_DmaDevice || !address || (address & 1U) || buffers[i].length > pageSize - offset) {
499 continue;
500 }
501 physical_uintptr_t physical = 0;
502 size_t flags = 0;
503 if (addressSpace.getMapping(reinterpret_cast<void*>(address - offset), physical, flags) &&
504 physical && !(physical & (pageSize - 1)) &&
505 (flags & (VirtualAddressSpace::KernelMode | VirtualAddressSpace::Write)) ==
506 (VirtualAddressSpace::KernelMode | VirtualAddressSpace::Write) &&
507 !(flags & (VirtualAddressSpace::Swapped | VirtualAddressSpace::CopyOnWrite |
508 VirtualAddressSpace::NoAccess | VirtualAddressSpace::Borrowed)) &&
509 PciBus::instance().mapDmaPage(m_DmaDevice, physical, pageSize, mappings[i])) {
510 dmaAddresses[i] = mappings[i].address() + offset;
511 }
512 }
513 const physical_uintptr_t* directReadPages[Disk::MaxReadBuffers] = {};
514 for (size_t i = 0; i < count; ++i) {
515 if (dmaAddresses[i]) {
516 directReadPages[i] = &dmaAddresses[i];
517 }
518 }
519 if (count < 2) {
520 return transferBatch(buffers, count, interrupts, false, nullptr, directReadPages);
521 }
522
523 // PRDs address individual pages, so direct reads retain grouped commands.
524 constexpr size_t ReadTransfer = 16 * 1024;
525 static_assert(ReadTransfer <= MaxTransfer);
526 Disk::ReadBuffer transfers[Disk::MaxReadBuffers];
527 const physical_uintptr_t* transferPages[Disk::MaxReadBuffers] = {};
528 size_t ends[Disk::MaxReadBuffers];
529 size_t grouped = 0;
530 for (size_t i = 0; i < count; ++i) {
531 const auto& buffer = buffers[i];
532 const bool direct = directReadPages[i] != nullptr;
533 const bool previousDirect = grouped && transferPages[grouped - 1] != nullptr;
534 if (grouped && direct == previousDirect && buffer.length == pageSize) {
535 auto& previous = transfers[grouped - 1];
536 const uintptr_t address = reinterpret_cast<uintptr_t>(buffer.buffer);
537 const uintptr_t priorAddress = reinterpret_cast<uintptr_t>(previous.buffer);
538 if (previous.length % pageSize == 0 && previous.length <= ReadTransfer &&
539 buffer.length <= ReadTransfer - previous.length && buffer.location > previous.location &&
540 buffer.location - previous.location == previous.length && address > priorAddress &&
541 address - priorAddress == previous.length) {
542 previous.length += buffer.length;
543 ends[grouped - 1] = i + 1;
544 continue;
545 }
546 }
547 transfers[grouped] = buffer;
548 transferPages[grouped] = directReadPages[i];
549 ends[grouped++] = i + 1;
550 }
551 const bool success = transferBatch(transfers, grouped, interrupts, false, nullptr, transferPages);
552 size_t next = 0;
553 for (size_t i = 0; i < grouped; ++i) {
554 while (next < ends[i]) {
555 buffers[next++].complete = transfers[i].complete;
556 }
557 }
558 return success;
559}
560
561bool AhciPort::writeBatch(Disk::WriteBuffer* buffers, size_t count, bool interrupts) {
562 static_assert(Disk::MaxWriteBuffers <= Disk::MaxReadBuffers);
563 if (count > Disk::MaxWriteBuffers || (count && !buffers))
564 return false;
565 Disk::ReadBuffer transfers[Disk::MaxWriteBuffers];
566 physical_uintptr_t dmaPhysical[Disk::MaxWriteBuffers] = {};
567 for (size_t i = 0; i < count; ++i) {
568 buffers[i].complete = false;
569 transfers[i] = {buffers[i].location, const_cast<void*>(buffers[i].buffer), buffers[i].length,
570 false};
571 dmaPhysical[i] = buffers[i].dmaPhysical;
572 }
573 const bool success = transferBatch(transfers, count, interrupts, true, dmaPhysical);
574 for (size_t i = 0; i < count; ++i)
575 buffers[i].complete = transfers[i].complete;
576 return success;
577}
578
579bool AhciPort::transferBatch(Disk::ReadBuffer* buffers, size_t count, bool interrupts, bool writing,
580 const physical_uintptr_t* dmaPhysical,
581 const physical_uintptr_t* const* directReadPages) {
582 if (count > Disk::MaxReadBuffers || (count && !buffers))
583 return false;
584 for (size_t i = 0; i < count; ++i)
585 buffers[i].complete = false;
586 for (size_t i = 0; i < count; ++i) {
587 const auto& buffer = buffers[i];
588 if (!buffer.buffer || !buffer.length ||
589 (writing && buffer.length > TargetInfo::getPageSize()) || buffer.length > MaxTransfer ||
590 !m_SectorBytes || buffer.location % m_SectorBytes || buffer.length % m_SectorBytes ||
591 buffer.location / m_SectorBytes >= (1ULL << 48) ||
592 buffer.length / m_SectorBytes > (1ULL << 48) - buffer.location / m_SectorBytes) {
593 return false;
594 }
595 }
596 TerminationDeferral lifetime;
597 if (!m_QueueDepth) {
598 for (size_t i = 0; i < count; ++i) {
599 auto& buffer = buffers[i];
600 buffer.complete =
601 command(writing ? 0x35 : 0x25, buffer.location / m_SectorBytes,
602 buffer.length / m_SectorBytes, buffer.buffer, buffer.length, writing, interrupts,
603 false, false, writing && dmaPhysical ? dmaPhysical[i] : 0,
604 !writing && directReadPages ? directReadPages[i] : nullptr);
605 if (!buffer.complete)
606 return false;
607 }
608 return true;
609 }
610
611 size_t next = 0;
612 while (next < count) {
613 size_t slots[Disk::MaxReadBuffers];
614 const size_t first = next;
615 size_t issued = 0;
616 bool admitted = true;
617 {
618 // A flush cannot overtake this wave. Reaping never requires this gate.
619 LockGuard<Mutex> command(m_CommandLock);
620 const auto admissionDeadline = Time::getTicks() + 120 * Time::Multiplier::Second;
621 while (next < count) {
622 size_t index = 32;
623 if (!chooseSlot(true, index)) {
624 admitted = false;
625 break;
626 }
627 if (index == 32) {
628 // Completed tags remain owned until reaped. Waiting here while owning
629 // tags could prevent this very batch from freeing the next slot.
630 if (issued)
631 break;
632 if (Time::getTicks() >= admissionDeadline) {
633 admitted = false;
634 break;
635 }
636 waitForProgress();
637 continue;
638 }
639 auto& buffer = buffers[next];
640 if (writing) {
641 m_WritesPending = true;
642 }
643 const physical_uintptr_t directPhysical =
644 writing && dmaPhysical ? usableDirectWrite(dmaPhysical[next], buffer.length) : 0;
645 if (!issueCommand(index, writing ? 0x61 : 0x60, buffer.location / m_SectorBytes,
646 buffer.length / m_SectorBytes, buffer.buffer, buffer.length, writing,
647 true, interrupts, directPhysical,
648 !writing && directReadPages ? directReadPages[next] : nullptr)) {
649 admitted = false;
650 break;
651 }
652 slots[issued++] = index;
653 ++next;
654 }
655 }
656 bool succeeded = admitted;
657 for (size_t i = 0; i < issued; ++i) {
658 auto& buffer = buffers[first + i];
659 buffer.complete =
660 reapCommand(slots[i], writing ? 0x61 : 0x60, buffer.buffer, buffer.length, writing, true,
661 interrupts, false, !writing && directReadPages && directReadPages[first + i]);
662 succeeded = buffer.complete && succeeded;
663 }
664 if (!succeeded)
665 return false;
666 }
667 return true;
668}
669
670void AhciPort::shutdown() {
671 auto reportShutdown = [&](const char* step) {
672 LargeStaticString detail("AHCI: port ");
673 detail.append(m_Port);
674 detail += ": ";
675 detail += step;
676 Machine::setShutdownDetail(detail);
677 };
678 reportShutdown("locking command state");
679 LockGuard<Mutex> command(m_CommandLock);
680 if (!m_AddressesInstalled)
681 return;
682 reportShutdown("draining active commands");
683 const auto deadline = Time::getTicks() + 120 * Time::Multiplier::Second;
684 for (;;) {
685 {
686 LockGuard<Mutex> state(m_StateLock);
687 if (!m_Active)
688 break;
689 }
690 if (Time::getTicks() >= deadline)
691 panic("AHCI: command owners did not drain during shutdown");
692 waitForProgress();
693 }
694 {
695 LockGuard<Mutex> state(m_StateLock);
696 m_Online = false;
697 write(PortIe, 0);
698 }
699 reportShutdown("stopping DMA engines");
700 if (!stopEngines())
701 panic("AHCI: cannot stop port DMA during shutdown");
702 reportShutdown("clearing DMA addresses");
703 write(Clb, 0);
704 write(Clbu, 0);
705 write(Fb, 0);
706 write(Fbu, 0);
707 acknowledge(read(PortIs));
708 (void)read(Cmd);
709 m_AddressesInstalled = false;
710}
711size_t AhciPort::interruptCompletions() const {
712 LockGuard<Mutex> state(m_StateLock);
713 return m_InterruptCompletions;
714}
715
716size_t AhciPort::maximumOutstanding() const {
717 LockGuard<Mutex> state(m_StateLock);
718 return m_MaximumOutstanding;
719}
IoBase
Abstrace base class for hardware I/O capabilities.
Definition IoBase.h:32
IoBase::write32
virtual void write32(uint32_t value, size_t offset=0)=0
IoBase::read32
virtual uint32_t read32(size_t offset=0)=0
Machine::setShutdownDetail
static void setShutdownDetail(const char *detail)
Definition Machine.cc:38
MemoryRegion::virtualAddress
void * virtualAddress() const
Definition MemoryRegion.cc:41
MemoryRegion::physicalAddress
physical_uintptr_t physicalAddress() const
Definition MemoryRegion.cc:45
MemoryRegion::size
size_t size() const
Definition MemoryRegion.cc:49
PciBus::mapDmaPage
bool mapDmaPage(Device *device, physical_uintptr_t physical, size_t bytes, DmaMapping &mapping)
Definition mach_pc/Pci.cc:435
PhysicalMemoryManager::instance
static PhysicalMemoryManager & instance()
Definition arm64/PhysicalMemoryManager.cc:34
ProcessorBase::getInterrupts
static bool getInterrupts()
Definition arm64/Processor.cc:157
ProcessorBase::information
static ProcessorInformation & information()
Definition x64/Processor.h:30
ProcessorBase::pause
static void pause()
Definition arm64/Processor.cc:180
Scheduler::instance
static Scheduler & instance()
Definition Scheduler.h:96
Scheduler::yield
void yield()
Definition Scheduler.cc:236
Semaphore::release
void release(size_t n=1)
Definition Semaphore.cc:549
TargetInfo::getPageSize
static constexpr size_t getPageSize() noexcept
Definition TargetInfo.h:40
Thread::eventsDeferred
bool eventsDeferred() const
Definition Thread.cc:3159
VirtualAddressSpace::getMapping
virtual bool getMapping(void *virtualAddress, physical_uintptr_t &physicalAddress, size_t &flags)=0
VirtualAddressSpace::getKernelAddressSpace
static EXPORTED_PUBLIC VirtualAddressSpace & getKernelAddressSpace()
Definition arm64/VirtualAddressSpace.cc:39
panic
void EXPORTED_PUBLIC panic(const char *msg) NORETURN
Definition panic.cc:118
Dec
@ Dec
Definition Log.h:126
Hex
@ Hex
Definition Log.h:124
Disk::WriteBuffer::dmaPhysical
physical_uintptr_t dmaPhysical
Definition Disk.h:132
cmd
Definition cmd.h:30
Generated on Sat Oct 3 2026 06:38:05 for The Pedigree Project by doxygen 1.9.8