8#include "pedigree/kernel/Atomic.h"
9#include "pedigree/kernel/Log.h"
10#include "pedigree/kernel/errors.h"
11#include "pedigree/kernel/machine/Disk.h"
12#include "pedigree/kernel/process/Process.h"
13#include "pedigree/kernel/process/Scheduler.h"
14#include "pedigree/kernel/process/Semaphore.h"
15#include "pedigree/kernel/process/Thread.h"
16#include "pedigree/kernel/processor/PhysicalMemoryManager.h"
17#include "pedigree/kernel/processor/Processor.h"
18#include "pedigree/kernel/processor/VirtualAddressSpace.h"
19#include "pedigree/kernel/utilities/utility.h"
24#include "modules/subsys/posix/FileDescriptor.h"
25#include "modules/subsys/posix/PosixSubsystem.h"
26#include "modules/subsys/posix/file-syscalls.h"
27#include "modules/system/vfs/File.h"
28#include "modules/system/vfs/Filesystem.h"
30#include "modules/system/vfs/Pipe.h"
33constexpr size_t BounceCapacity = PIPE_BUF_MAX + 1;
34constexpr size_t FirstVectorLength = BounceCapacity + 5;
35constexpr size_t SecondVectorLength = BounceCapacity - 3;
36constexpr size_t VectorWriteLength = FirstVectorLength + SecondVectorLength;
37constexpr int PreservedErrno = 123;
39bool closeDescriptor(
PosixSubsystem* subsystem,
size_t descriptor) {
45bool pointsInto(uintptr_t pointer,
const void* base,
size_t length) {
46 const uintptr_t start =
reinterpret_cast<uintptr_t
>(base);
47 return length && pointer >= start && pointer < start + length;
50class VectorWriteProbeFile final :
public File {
52 VectorWriteProbeFile()
53 :
File(
String(
"vector-write-bounce"), 0, 0, 0, 1, nullptr, 0, nullptr),
54 m_FirstWriteEntered(0, false),
55 m_ReleaseFirstWrite(0, false),
56 m_FirstSource(nullptr),
57 m_SecondSource(nullptr),
60 m_SecondChunkBeforeBoundary(0),
61 m_SecondChunkAfterBoundary(0),
62 m_Offsets{0, 0, 0, 0},
64 m_FirstValues{0, 0, 0, 0} {}
66 void setSources(
char* first,
char* second) {
67 m_FirstSource = first;
68 m_SecondSource = second;
71 bool waitForFirstWrite() {
72 return m_FirstWriteEntered.acquireForCompletion();
75 void releaseFirstWrite() {
76 m_ReleaseFirstWrite.release();
79 size_t writeCount()
const {
83 uint64_t offset(
size_t index)
const {
84 return m_Offsets[index];
87 size_t size(
size_t index)
const {
88 return m_Sizes[index];
91 char firstValue(
size_t index)
const {
92 return m_FirstValues[index];
95 bool sawRawPointer()
const {
99 char secondChunkBeforeBoundary()
const {
100 return m_SecondChunkBeforeBoundary;
103 char secondChunkAfterBoundary()
const {
104 return m_SecondChunkAfterBoundary;
112 uint64_t
writeBytewise(uint64_t location, uint64_t size, uintptr_t buffer,
bool)
override {
113 const size_t slot = (m_WriteCount += 1) - 1;
115 m_Offsets[slot] = location;
116 m_Sizes[slot] = size;
117 m_FirstValues[slot] = size ? *
reinterpret_cast<const char*
>(buffer) : 0;
119 m_RawPointer = m_RawPointer || pointsInto(buffer, m_FirstSource, FirstVectorLength) ||
120 pointsInto(buffer, m_SecondSource, SecondVectorLength);
121 if (slot == 1 && size > 5) {
122 m_SecondChunkBeforeBoundary =
reinterpret_cast<const char*
>(buffer)[4];
123 m_SecondChunkAfterBoundary =
reinterpret_cast<const char*
>(buffer)[5];
128 ByteSet(m_FirstSource,
'z', FirstVectorLength);
129 ByteSet(m_SecondSource,
'y', SecondVectorLength);
130 m_FirstWriteEntered.release();
131 if (!m_ReleaseFirstWrite.acquireForCompletion()) {
136 if (location + size > getSize()) {
137 setSize(location + size);
146 char* m_SecondSource;
149 char m_SecondChunkBeforeBoundary;
150 char m_SecondChunkAfterBoundary;
151 uint64_t m_Offsets[4];
153 char m_FirstValues[4];
156class VectorReplacementFile final :
public File {
158 VectorReplacementFile()
159 :
File(
String(
"vector-write-replacement"), 0, 0, 0, 2, nullptr, 0, nullptr), m_Writes(0) {}
161 size_t writes()
const {
170 uint64_t
writeBytewise(uint64_t, uint64_t size, uintptr_t,
bool)
override {
179struct VectorWriteContext {
180 VectorWriteContext(
size_t descriptor, VectorWriteProbeFile* file)
181 : descriptor(descriptor), file(file), entered(0), result(-2), error(0), returned(0) {}
184 VectorWriteProbeFile* file;
191int chunkedVectorWrite(
void* parameter) {
192 VectorWriteContext* context =
reinterpret_cast<VectorWriteContext*
>(parameter);
193 char first[FirstVectorLength];
194 char second[SecondVectorLength];
195 ByteSet(first,
'a',
sizeof(first));
196 ByteSet(second,
'b',
sizeof(second));
197 context->file->setSources(first, second);
198 context->entered += 1;
200 struct iovec vectors[2] = {{first,
sizeof(first)}, {second,
sizeof(second)}};
203 context->result = posix_writev(
static_cast<int>(context->descriptor), vectors, 2);
204 context->error = thread->
getErrno();
205 context->returned += 1;
209int vectorAliasWrite(
void* parameter) {
210 VectorWriteContext* context =
reinterpret_cast<VectorWriteContext*
>(parameter);
212 context->entered += 1;
215 context->result = posix_write(
static_cast<int>(context->descriptor), &value, 1,
false);
216 context->error = thread->
getErrno();
217 context->returned += 1;
221bool vectorWriteBounceAndLifetime(
Process* kernelProcess) {
222 constexpr size_t SourceDescriptor = 90;
223 constexpr size_t AliasDescriptor = 91;
227 process->setSubsystem(subsystem);
228 VectorWriteProbeFile sourceFile;
229 VectorReplacementFile replacementFile;
232 alias->
fd = AliasDescriptor;
237 VectorWriteContext sourceContext(SourceDescriptor, &sourceFile);
238 VectorWriteContext aliasContext(AliasDescriptor, &sourceFile);
240 new Thread(process, chunkedVectorWrite, &sourceContext,
nullptr,
false,
true,
true);
242 new Thread(process, vectorAliasWrite, &aliasContext,
nullptr,
false,
true,
true);
243 sourceWorker->setName(
"hosted vector bounce writer");
244 aliasWorker->setName(
"hosted vector bounce alias");
246 const bool sourceStarted = sourceWorker->
start();
247 const bool firstEntered = sourceStarted && sourceFile.waitForFirstWrite();
248 const bool aliasStarted = firstEntered && aliasWorker->
start();
249 while (aliasStarted && !aliasContext.entered) {
252 for (
size_t attempt = 0; attempt < 32 && aliasStarted && !aliasContext.returned; ++attempt) {
255 const bool aliasWasSerialized = aliasStarted && !aliasContext.returned;
257 const bool sourceClosed = firstEntered && closeDescriptor(subsystem, SourceDescriptor);
259 SourceDescriptor,
new FileDescriptor(&replacementFile, 0, SourceDescriptor, 0, O_WRONLY));
261 sourceFile.releaseFirstWrite();
264 if (!sourceStarted) {
271 bool passed = sourceStarted && firstEntered && aliasStarted && aliasWasSerialized &&
272 sourceClosed && sourceJoined && aliasJoined && sourceContext.returned == 1 &&
273 sourceContext.result ==
static_cast<int>(VectorWriteLength) &&
274 sourceContext.error == PreservedErrno && aliasContext.returned == 1 &&
275 aliasContext.result == 1 && aliasContext.error == PreservedErrno &&
276 !sourceFile.sawRawPointer() && sourceFile.writeCount() == 4 &&
277 sourceFile.offset(0) == 0 && sourceFile.size(0) == BounceCapacity &&
278 sourceFile.firstValue(0) ==
'a' && sourceFile.offset(1) == BounceCapacity &&
279 sourceFile.size(1) == BounceCapacity && sourceFile.firstValue(1) ==
'z' &&
280 sourceFile.secondChunkBeforeBoundary() ==
'z' &&
281 sourceFile.secondChunkAfterBoundary() ==
'y' &&
282 sourceFile.offset(2) == BounceCapacity * 2 && sourceFile.size(2) == 2 &&
283 sourceFile.firstValue(2) ==
'y' && sourceFile.offset(3) == VectorWriteLength &&
284 sourceFile.size(3) == 1 && sourceFile.firstValue(3) ==
'q' &&
285 replacementFile.writes() == 0 && alias->
getOffset() == VectorWriteLength + 1;
287 passed = closeDescriptor(subsystem, AliasDescriptor) &&
288 closeDescriptor(subsystem, SourceDescriptor) && passed;
294 "HOSTED-SYSCALL-TEST: FAIL vector-write-bounce-lifetime: "
295 "writev exposed a user pointer, interleaved chunks, or switched descriptor generations");
299 NOTICE(
"HOSTED-SYSCALL-TEST: PASS vector-write-bounce-lifetime");
303char vectorPattern(
size_t offset) {
304 return static_cast<char>(
'A' + (offset % 23));
307class VectorReadProbeFile final :
public File {
309 VectorReadProbeFile()
310 :
File(
String(
"vector-read-bounce"), 0, 0, 0, 3, nullptr, 0, nullptr),
311 m_FirstDestination(nullptr),
313 m_SecondDestination(nullptr),
320 void setDestinations(
void* first,
size_t firstLength,
void* second,
size_t secondLength) {
321 m_FirstDestination = first;
322 m_FirstLength = firstLength;
323 m_SecondDestination = second;
324 m_SecondLength = secondLength;
327 size_t readCount()
const {
331 bool sawRawPointer()
const {
335 uint64_t offset(
size_t index)
const {
336 return m_Offsets[index];
339 size_t size(
size_t index)
const {
340 return m_Sizes[index];
348 uint64_t
readBytewise(uint64_t location, uint64_t size, uintptr_t buffer,
bool)
override {
349 const size_t slot = (m_ReadCount += 1) - 1;
351 m_Offsets[slot] = location;
352 m_Sizes[slot] = size;
354 m_RawPointer = m_RawPointer || pointsInto(buffer, m_FirstDestination, m_FirstLength) ||
355 pointsInto(buffer, m_SecondDestination, m_SecondLength);
356 for (
size_t i = 0; i < size; ++i) {
357 reinterpret_cast<char*
>(buffer)[i] = vectorPattern(location + i);
363 void* m_FirstDestination;
364 size_t m_FirstLength;
365 void* m_SecondDestination;
366 size_t m_SecondLength;
369 uint64_t m_Offsets[3];
373struct VectorReadContext {
374 VectorReadContext(
size_t descriptor, VectorReadProbeFile* file)
375 : descriptor(descriptor),
377 contentsValid(false),
383 VectorReadProbeFile* file;
390int chunkedVectorRead(
void* parameter) {
391 constexpr size_t FirstLength = BounceCapacity + 7;
392 constexpr size_t SecondLength = 13;
393 VectorReadContext* context =
reinterpret_cast<VectorReadContext*
>(parameter);
394 char first[FirstLength];
395 char second[SecondLength];
396 ByteSet(first, 0,
sizeof(first));
397 ByteSet(second, 0,
sizeof(second));
398 context->file->setDestinations(first,
sizeof(first), second,
sizeof(second));
400 struct iovec vectors[2] = {{first,
sizeof(first)}, {second,
sizeof(second)}};
403 context->result = posix_readv(
static_cast<int>(context->descriptor), vectors, 2);
404 context->error = thread->
getErrno();
407 for (
size_t i = 0; i <
sizeof(first) && valid; ++i) {
408 valid = first[i] == vectorPattern(i);
410 for (
size_t i = 0; i <
sizeof(second) && valid; ++i) {
411 valid = second[i] == vectorPattern(
sizeof(first) + i);
413 context->contentsValid = valid;
414 context->returned += 1;
418bool vectorReadBounceAndScatter(
Process* kernelProcess) {
419 constexpr size_t Descriptor = 92;
420 constexpr size_t FirstLength = BounceCapacity + 7;
421 constexpr size_t TotalLength = FirstLength + 13;
425 process->setSubsystem(subsystem);
426 VectorReadProbeFile file;
431 VectorReadContext context(Descriptor, &file);
432 Thread*
worker =
new Thread(process, chunkedVectorRead, &context,
nullptr,
false,
true,
true);
433 worker->setName(
"hosted vector bounce reader");
434 const bool started =
worker->start();
435 const bool joined = started &&
worker->joinForCompletion();
440 bool passed = started && joined && context.returned == 1 &&
441 context.result ==
static_cast<int>(TotalLength) &&
442 context.error == PreservedErrno && context.contentsValid && !file.sawRawPointer() &&
443 file.readCount() == 3 && file.offset(0) == 0 && file.size(0) == BounceCapacity &&
444 file.offset(1) == BounceCapacity && file.size(1) == 7 &&
445 file.offset(2) == FirstLength && file.size(2) == 13 &&
447 passed = closeDescriptor(subsystem, Descriptor) && passed;
453 "HOSTED-SYSCALL-TEST: FAIL vector-read-bounce-scatter: "
454 "readv exposed a user pointer or scattered bounce chunks incorrectly");
458 NOTICE(
"HOSTED-SYSCALL-TEST: PASS vector-read-bounce-scatter");
462bool allocateUserMapping(
Process* process,
size_t length, uintptr_t& address) {
464 if (!process->allocateUserRange(Process::UserRegion::Normal, length, address)) {
468 uintptr_t mappedAddress = address;
470 mappedAddress, length, MemoryMappedObject::Read | MemoryMappedObject::Write);
471 if (!mapping || mappedAddress != address) {
473 process->freeUserRange(Process::UserRegion::Normal, address, length);
480class VectorReadDisk final :
public Disk {
482 bool pin(uint64_t)
override {
485 void unpin(uint64_t)
override {}
488class VectorReadFilesystem final :
public Filesystem {
519constexpr size_t DiskReadCapacity = 64 * 1024;
520enum class DiskVectorReadMode { Complete, Short, PrefixFault, CopyFault };
522class DiskVectorReadFile final :
public File {
524 explicit DiskVectorReadFile(
Filesystem* filesystem)
525 :
File(
String(
"disk-vector-read"), 0, 0, 0, 3, filesystem, DiskReadCapacity * 3, nullptr),
526 m_Data(
UniqueArray<char>::allocate(DiskReadCapacity * 3)) {
527 for (
size_t i = 0; i < DiskReadCapacity * 3; ++i) {
528 m_Data.get()[i] = vectorPattern(i);
532 size_t readCount()
const {
536 void denyWritesAfterFirstRead(uintptr_t address,
size_t length) {
537 m_DenyAddress = address;
538 m_DenyLength = length;
542 Mutex& dataMutationLock()
override {
544 if (!m_ReadCount++ && m_DenyLength) {
546 MemoryMappedObject::Read);
548 return File::dataMutationLock();
551 uintptr_t
readBlock(uint64_t location)
override {
552 return reinterpret_cast<uintptr_t
>(m_Data.get() + location);
561 size_t m_ReadCount = 0;
562 uintptr_t m_DenyAddress = 0;
563 size_t m_DenyLength = 0;
566struct DiskVectorReadContext {
568 DiskVectorReadFile* file;
569 DiskVectorReadMode mode;
576int diskVectorReader(
void* parameter) {
577 auto& context = *
static_cast<DiskVectorReadContext*
>(parameter);
579 const size_t mappingLength = DiskReadCapacity * 2 + pageSize * 3;
580 uintptr_t address = 0;
581 if (!allocateUserMapping(context.process, mappingLength, address)) {
584 auto* first =
reinterpret_cast<char*
>(address);
585 const bool prefixFault = context.mode == DiskVectorReadMode::PrefixFault;
586 const size_t firstLength = prefixFault ? 13 : DiskReadCapacity + 7;
587 const size_t secondLength = prefixFault ? DiskReadCapacity * 2 : 13;
588 auto* second = first + (prefixFault ? pageSize : DiskReadCapacity + pageSize);
589 ByteSet(first, 0, mappingLength);
590 if (context.mode == DiskVectorReadMode::Short) {
591 context.file->setSize((context.positional ? 29 : 17) + 23);
592 }
else if (prefixFault) {
595 const uintptr_t denied =
reinterpret_cast<uintptr_t
>(second) + pageSize * 2;
596 context.file->denyWritesAfterFirstRead(denied, address + mappingLength - denied);
597 }
else if (context.mode == DiskVectorReadMode::CopyFault) {
598 const uintptr_t denied = address + pageSize * 2;
599 context.file->denyWritesAfterFirstRead(denied, address + mappingLength - denied);
602 struct iovec vectors[2] = {{first, firstLength}, {second, secondLength}};
606 context.positional ? posix_preadv(92, vectors, 2, 29) : posix_readv(92, vectors, 2);
609 const size_t firstCopied = context.mode == DiskVectorReadMode::CopyFault ? 0
610 : context.mode == DiskVectorReadMode::Short ? 23
612 const size_t secondCopied = prefixFault ? BounceCapacity
613 : context.mode == DiskVectorReadMode::Complete ? secondLength
615 const size_t start = context.positional ? 29 : 17;
617 for (
size_t i = 0; i < firstLength && valid; ++i) {
618 valid = first[i] == (i < firstCopied ? vectorPattern(start + i) : 0);
620 for (
size_t i = 0; i < secondLength && valid; ++i) {
621 valid = second[i] == (i < secondCopied ? vectorPattern(start + firstCopied + i) : 0);
623 context.valid = valid;
625 context.process->freeUserRange(Process::UserRegion::Normal, address, mappingLength);
629bool diskVectorReadCase(
Process* kernelProcess, DiskVectorReadMode mode,
bool positional) {
632 process->setSubsystem(subsystem);
634 VectorReadFilesystem filesystem;
635 filesystem.initialise(&disk);
636 DiskVectorReadFile file(&filesystem);
637 auto* descriptor =
new FileDescriptor(&file, 17, 92, 0, O_RDONLY);
641 DiskVectorReadContext context{process, &file, mode, positional};
642 Thread*
worker =
new Thread(process, diskVectorReader, &context,
nullptr,
false,
true,
true);
643 worker->setName(
"hosted disk vector reader");
644 const bool started =
worker->start();
645 const bool joined = started &&
worker->joinForCompletion();
650 const ssize_t expected = mode == DiskVectorReadMode::CopyFault ? -1
651 : mode == DiskVectorReadMode::Short ? 23
652 : mode == DiskVectorReadMode::PrefixFault ? 13 + BounceCapacity
653 : DiskReadCapacity + 20;
654 const size_t calls = mode == DiskVectorReadMode::Complete ? 3
655 : mode == DiskVectorReadMode::PrefixFault ? 2
657 const uint64_t finalOffset = positional || expected < 0 ? 17 : 17 + expected;
658 bool passed = started && joined && context.valid && context.result == expected &&
659 context.error == (expected < 0 ? Error::BadAddress : PreservedErrno) &&
660 file.readCount() == calls && descriptor->
getOffset() == finalOffset;
661 passed = closeDescriptor(subsystem, 92) && passed;
665 ERROR(
"HOSTED-SYSCALL-TEST: FAIL disk-vector-read-batching: mode="
666 <<
static_cast<int>(mode) <<
" positional=" << positional <<
" result=" << context.result
667 <<
" errno=" << context.error <<
" calls=" << file.readCount());
672bool diskVectorReadBatching(
Process* kernelProcess) {
673 const bool positions[] = {
false,
true};
674 const DiskVectorReadMode modes[] = {DiskVectorReadMode::Complete, DiskVectorReadMode::Short,
675 DiskVectorReadMode::PrefixFault,
676 DiskVectorReadMode::CopyFault};
677 for (
bool positional : positions) {
678 for (DiskVectorReadMode mode : modes) {
679 if (!diskVectorReadCase(kernelProcess, mode, positional)) {
684 NOTICE(
"HOSTED-SYSCALL-TEST: PASS disk-vector-read-batching");
688class FaultingVectorFile final :
public File {
690 explicit FaultingVectorFile(
bool reading)
691 :
File(
String(reading ?
"vector-read-fault" :
"vector-write-fault"), 0, 0, 0, reading ? 4 : 5,
692 nullptr, 0, nullptr),
696 m_RawPointer(false) {}
698 void configure(uintptr_t userBase,
size_t pageSize) {
699 m_UserBase = userBase;
700 m_PageSize = pageSize;
703 size_t calls()
const {
707 bool sawRawPointer()
const {
716 uint64_t
readBytewise(uint64_t, uint64_t size, uintptr_t buffer,
bool)
override {
717 const size_t slot = (m_Calls += 1) - 1;
719 m_RawPointer || pointsInto(buffer,
reinterpret_cast<void*
>(m_UserBase), m_PageSize * 2);
722 MemoryMappedObject::Read);
724 ByteSet(
reinterpret_cast<void*
>(buffer),
'r', size);
728 uint64_t
writeBytewise(uint64_t location, uint64_t size, uintptr_t buffer,
bool)
override {
729 const size_t slot = (m_Calls += 1) - 1;
731 m_RawPointer || pointsInto(buffer,
reinterpret_cast<void*
>(m_UserBase), m_PageSize * 2);
734 MemoryMappedObject::None);
736 if (location + size > getSize()) {
737 setSize(location + size);
743 uintptr_t m_UserBase;
749struct VectorFaultContext {
750 VectorFaultContext(
Process* process,
size_t writeFd,
size_t readFd, FaultingVectorFile* writeFile,
751 FaultingVectorFile* readFile)
755 writeFile(writeFile),
758 firstReadPageValid(false),
761 firstWriteFaultResult(-2),
762 firstWriteFaultError(0),
765 firstReadFaultResult(-2),
766 firstReadFaultError(0),
767 validationSemantics(false),
773 FaultingVectorFile* writeFile;
774 FaultingVectorFile* readFile;
776 bool firstReadPageValid;
779 int firstWriteFaultResult;
780 int firstWriteFaultError;
783 int firstReadFaultResult;
784 int firstReadFaultError;
785 bool validationSemantics;
789int vectorFaultWorker(
void* parameter) {
790 VectorFaultContext* context =
reinterpret_cast<VectorFaultContext*
>(parameter);
793 const size_t mappingLength = pageSize * 2;
795 uintptr_t writeAddress = 0;
796 if (!allocateUserMapping(context->process, mappingLength, writeAddress)) {
797 context->returned += 1;
800 ByteSet(
reinterpret_cast<void*
>(writeAddress),
'w', mappingLength);
801 context->writeFile->configure(writeAddress, pageSize);
802 struct iovec writeVectors[2] = {{
reinterpret_cast<void*
>(writeAddress), pageSize},
803 {
reinterpret_cast<void*
>(writeAddress + pageSize), pageSize}};
805 context->writeResult = posix_writev(
static_cast<int>(context->writeFd), writeVectors, 2);
806 context->writeError = thread->
getErrno();
809 struct iovec badVector = {
reinterpret_cast<void*
>(kernelStart), 1};
811 context->firstWriteFaultResult = posix_writev(
static_cast<int>(context->writeFd), &badVector, 1);
812 context->firstWriteFaultError = thread->
getErrno();
814 context->process->freeUserRange(Process::UserRegion::Normal, writeAddress, mappingLength);
816 const size_t progressMappingLength = pageSize * 3;
817 uintptr_t writeProgressAddress = 0;
818 if (!allocateUserMapping(context->process, progressMappingLength, writeProgressAddress)) {
819 context->returned += 1;
822 ByteSet(
reinterpret_cast<void*
>(writeProgressAddress),
'p', progressMappingLength);
824 MemoryMappedObject::None);
825 context->writeFile->configure(writeProgressAddress, pageSize);
826 struct iovec writeProgressVectors[2] = {
827 {
reinterpret_cast<void*
>(writeProgressAddress), BounceCapacity},
828 {
reinterpret_cast<void*
>(writeProgressAddress + pageSize * 2), 1}};
830 const int writeProgressResult =
831 posix_writev(
static_cast<int>(context->writeFd), writeProgressVectors, 2);
832 const int writeProgressError = thread->
getErrno();
834 struct iovec inaccessibleWriteVector = {
835 reinterpret_cast<void*
>(writeProgressAddress + pageSize * 2), 1};
837 const int inaccessibleWriteResult =
838 posix_writev(
static_cast<int>(context->writeFd), &inaccessibleWriteVector, 1);
839 const int inaccessibleWriteError = thread->
getErrno();
841 struct iovec laterInvalidWriteVectors[2] = {{
reinterpret_cast<void*
>(writeProgressAddress), 1},
842 {
reinterpret_cast<void*
>(kernelStart), 1}};
844 const int laterInvalidWriteResult =
845 posix_writev(
static_cast<int>(context->writeFd), laterInvalidWriteVectors, 2);
846 const int laterInvalidWriteError = thread->
getErrno();
848 const struct iovec* invalidVectorArray =
reinterpret_cast<const struct iovec*
>(kernelStart);
850 const int invalidWriteArrayResult =
851 posix_writev(
static_cast<int>(context->writeFd), invalidVectorArray, 1);
852 const int invalidWriteArrayError = thread->
getErrno();
854 struct iovec zeroLengthInvalidVector = {
reinterpret_cast<void*
>(kernelStart), 0};
856 const int zeroLengthWriteResult =
857 posix_writev(
static_cast<int>(context->writeFd), &zeroLengthInvalidVector, 1);
858 const int zeroLengthWriteError = thread->
getErrno();
861 const int badWriteDescriptorResult = posix_writev(-1, invalidVectorArray, 1);
862 const int badWriteDescriptorError = thread->
getErrno();
864 bool validationSemantics =
865 writeProgressResult ==
static_cast<int>(BounceCapacity) &&
866 writeProgressError == PreservedErrno && inaccessibleWriteResult == -1 &&
867 inaccessibleWriteError == Error::BadAddress && laterInvalidWriteResult == -1 &&
868 laterInvalidWriteError == Error::BadAddress && invalidWriteArrayResult == -1 &&
869 invalidWriteArrayError == Error::BadAddress && zeroLengthWriteResult == 0 &&
870 zeroLengthWriteError == PreservedErrno && badWriteDescriptorResult == -1 &&
871 badWriteDescriptorError == Error::BadFileDescriptor;
874 context->process->freeUserRange(Process::UserRegion::Normal, writeProgressAddress,
875 progressMappingLength);
877 uintptr_t readAddress = 0;
878 if (!allocateUserMapping(context->process, mappingLength, readAddress)) {
879 context->returned += 1;
882 ByteSet(
reinterpret_cast<void*
>(readAddress), 0, mappingLength);
883 context->readFile->configure(readAddress, pageSize);
884 struct iovec readVectors[2] = {{
reinterpret_cast<void*
>(readAddress), pageSize},
885 {
reinterpret_cast<void*
>(readAddress + pageSize), pageSize}};
887 context->readResult = posix_readv(
static_cast<int>(context->readFd), readVectors, 2);
888 context->readError = thread->
getErrno();
889 bool firstPageValid =
true;
890 for (
size_t i = 0; i < pageSize && firstPageValid; ++i) {
891 firstPageValid =
reinterpret_cast<char*
>(readAddress)[i] ==
'r';
893 context->firstReadPageValid = firstPageValid;
896 context->firstReadFaultResult = posix_readv(
static_cast<int>(context->readFd), &badVector, 1);
897 context->firstReadFaultError = thread->
getErrno();
899 context->process->freeUserRange(Process::UserRegion::Normal, readAddress, mappingLength);
901 uintptr_t readProgressAddress = 0;
902 if (!allocateUserMapping(context->process, progressMappingLength, readProgressAddress)) {
903 context->returned += 1;
906 ByteSet(
reinterpret_cast<void*
>(readProgressAddress), 0, progressMappingLength);
908 MemoryMappedObject::Read);
909 context->readFile->configure(readProgressAddress, pageSize);
910 struct iovec readProgressVectors[2] = {
911 {
reinterpret_cast<void*
>(readProgressAddress), pageSize},
912 {
reinterpret_cast<void*
>(readProgressAddress + pageSize * 2), 1}};
914 const int readProgressResult =
915 posix_readv(
static_cast<int>(context->readFd), readProgressVectors, 2);
916 const int readProgressError = thread->
getErrno();
917 bool readProgressContentsValid =
true;
918 for (
size_t i = 0; i < pageSize && readProgressContentsValid; ++i) {
919 readProgressContentsValid =
reinterpret_cast<char*
>(readProgressAddress)[i] ==
'r';
922 struct iovec inaccessibleReadVector = {
923 reinterpret_cast<void*
>(readProgressAddress + pageSize * 2), 1};
925 const int inaccessibleReadResult =
926 posix_readv(
static_cast<int>(context->readFd), &inaccessibleReadVector, 1);
927 const int inaccessibleReadError = thread->
getErrno();
929 struct iovec laterInvalidReadVectors[2] = {{
reinterpret_cast<void*
>(readProgressAddress), 1},
930 {
reinterpret_cast<void*
>(kernelStart), 1}};
932 const int laterInvalidReadResult =
933 posix_readv(
static_cast<int>(context->readFd), laterInvalidReadVectors, 2);
934 const int laterInvalidReadError = thread->
getErrno();
937 const int invalidReadArrayResult =
938 posix_readv(
static_cast<int>(context->readFd), invalidVectorArray, 1);
939 const int invalidReadArrayError = thread->
getErrno();
942 const int zeroLengthReadResult =
943 posix_readv(
static_cast<int>(context->readFd), &zeroLengthInvalidVector, 1);
944 const int zeroLengthReadError = thread->
getErrno();
947 const int badReadDescriptorResult = posix_readv(-1, invalidVectorArray, 1);
948 const int badReadDescriptorError = thread->
getErrno();
950 validationSemantics =
951 readProgressResult ==
static_cast<int>(pageSize) && readProgressError == PreservedErrno &&
952 readProgressContentsValid && inaccessibleReadResult == -1 &&
953 inaccessibleReadError == Error::BadAddress && laterInvalidReadResult == -1 &&
954 laterInvalidReadError == Error::BadAddress && invalidReadArrayResult == -1 &&
955 invalidReadArrayError == Error::BadAddress && zeroLengthReadResult == 0 &&
956 zeroLengthReadError == PreservedErrno && badReadDescriptorResult == -1 &&
957 badReadDescriptorError == Error::BadFileDescriptor && validationSemantics;
960 context->process->freeUserRange(Process::UserRegion::Normal, readProgressAddress,
961 progressMappingLength);
963 context->validationSemantics = validationSemantics;
964 context->setup =
true;
965 context->returned += 1;
969bool vectorUsercopyFaultProgress(
Process* kernelProcess) {
970 constexpr size_t WriteDescriptor = 93;
971 constexpr size_t ReadDescriptor = 94;
975 process->setSubsystem(subsystem);
976 FaultingVectorFile writeFile(
false);
977 FaultingVectorFile readFile(
true);
985 VectorFaultContext context(process, WriteDescriptor, ReadDescriptor, &writeFile, &readFile);
986 Thread*
worker =
new Thread(process, vectorFaultWorker, &context,
nullptr,
false,
true,
true);
987 worker->setName(
"hosted vector usercopy fault progress");
988 const bool started =
worker->start();
989 const bool joined = started &&
worker->joinForCompletion();
995 bool passed = started && joined && context.returned == 1 && context.setup &&
996 context.writeResult ==
static_cast<int>(BounceCapacity) &&
997 context.writeError == PreservedErrno && context.firstWriteFaultResult == -1 &&
998 context.firstWriteFaultError == Error::BadAddress && writeFile.calls() == 2 &&
999 !writeFile.sawRawPointer() && writer->
getOffset() == BounceCapacity * 2 &&
1000 context.readResult ==
static_cast<int>(pageSize) &&
1001 context.readError == PreservedErrno && context.firstReadPageValid &&
1002 context.firstReadFaultResult == -1 &&
1003 context.firstReadFaultError == Error::BadAddress && readFile.calls() == 3 &&
1004 !readFile.sawRawPointer() &&
reader->getOffset() == pageSize * 2 &&
1005 context.validationSemantics;
1006 passed = closeDescriptor(subsystem, WriteDescriptor) &&
1007 closeDescriptor(subsystem, ReadDescriptor) && passed;
1008 writeDescription.
reset();
1009 readDescription.
reset();
1014 "HOSTED-SYSCALL-TEST: FAIL vector-usercopy-fault-progress: "
1015 "writev/readv validation, partial progress, or offsets regressed");
1019 NOTICE(
"HOSTED-SYSCALL-TEST: PASS vector-usercopy-fault-progress");
1023struct VectorPipeFaultContext {
1024 VectorPipeFaultContext(
Process* process,
size_t readFd,
size_t writeFd)
1046int vectorPipeFaultWorker(
void* parameter) {
1047 VectorPipeFaultContext* context =
reinterpret_cast<VectorPipeFaultContext*
>(parameter);
1050 const size_t mappingLength = pageSize * 2;
1051 uintptr_t address = 0;
1052 if (!allocateUserMapping(context->process, mappingLength, address)) {
1053 context->returned += 1;
1057 ByteSet(
reinterpret_cast<void*
>(address),
'v', mappingLength);
1059 MemoryMappedObject::None);
1060 struct iovec writeVectors[2] = {{
reinterpret_cast<void*
>(address), 1},
1061 {
reinterpret_cast<void*
>(address + pageSize), 1}};
1063 context->writeResult = posix_writev(
static_cast<int>(context->writeFd), writeVectors, 2);
1064 context->writeError = thread->
getErrno();
1066 struct iovec readVectors[2] = {{
reinterpret_cast<void*
>(address + 16), 1},
1067 {
reinterpret_cast<void*
>(address + pageSize), 1}};
1069 context->readResult = posix_readv(
static_cast<int>(context->readFd), readVectors, 2);
1070 context->readError = thread->
getErrno();
1073 context->process->freeUserRange(Process::UserRegion::Normal, address, mappingLength);
1074 context->setup =
true;
1075 context->returned += 1;
1079bool vectorPipePayloadPrevalidation(
Process* kernelProcess) {
1080 constexpr size_t ReadDescriptor = 99;
1081 constexpr size_t WriteDescriptor = 100;
1085 process->setSubsystem(subsystem);
1092 char contents[2] = {
'x',
'y'};
1093 const bool filled = writer->
write(
sizeof(contents),
reinterpret_cast<uintptr_t
>(contents),
1094 true) ==
sizeof(contents);
1095 VectorPipeFaultContext context(process, ReadDescriptor, WriteDescriptor);
1096 Thread*
worker =
new Thread(process, vectorPipeFaultWorker, &context,
nullptr,
false,
true,
true);
1097 worker->setName(
"hosted vector pipe payload prevalidation");
1098 const bool started = filled &&
worker->start();
1099 const bool joined = started &&
worker->joinForCompletion();
1104 char drained[4] = {};
1105 const uint64_t drainResult =
1106 joined ?
reader->read(
sizeof(drained),
reinterpret_cast<uintptr_t
>(drained),
false) : 0;
1107 bool passed = started && joined && context.returned == 1 && context.setup &&
1108 context.writeResult == -1 && context.writeError == Error::BadAddress &&
1109 context.readResult == -1 && context.readError == Error::BadAddress &&
1110 drainResult ==
sizeof(contents) && drained[0] ==
'x' && drained[1] ==
'y';
1111 passed = closeDescriptor(subsystem, WriteDescriptor) &&
1112 closeDescriptor(subsystem, ReadDescriptor) && passed;
1117 "HOSTED-SYSCALL-TEST: FAIL vector-pipe-payload-prevalidation: "
1118 "faulting vectors changed pipe contents");
1122 NOTICE(
"HOSTED-SYSCALL-TEST: PASS vector-pipe-payload-prevalidation");
1126class InterruptingVectorFile final :
public File {
1128 InterruptingVectorFile()
1129 :
File(
String(
"vector-interruption"), 0, 0, 0, 6, nullptr, 0, nullptr),
1130 m_MakeProgress(false),
1133 void setMakeProgress(
bool makeProgress) {
1134 m_MakeProgress = makeProgress;
1137 size_t calls()
const {
1146 uint64_t
writeBytewise(uint64_t location, uint64_t size, uintptr_t,
bool)
override {
1149 if (!m_MakeProgress) {
1152 const size_t amount = size < 3 ? size : 3;
1153 if (location + amount > getSize()) {
1154 setSize(location + amount);
1160 bool m_MakeProgress;
1164struct VectorSignalContext {
1165 VectorSignalContext(
size_t descriptor, InterruptingVectorFile* file)
1166 : descriptor(descriptor),
1168 noProgressResult(-2),
1175 InterruptingVectorFile* file;
1176 int noProgressResult;
1177 int noProgressError;
1183int vectorSignalWorker(
void* parameter) {
1184 VectorSignalContext* context =
reinterpret_cast<VectorSignalContext*
>(parameter);
1185 char first[3] = {
'a',
'b',
'c'};
1186 char second[2] = {
'd',
'e'};
1187 struct iovec vectors[2] = {{first,
sizeof(first)}, {second,
sizeof(second)}};
1190 context->file->setMakeProgress(
false);
1192 context->noProgressResult = posix_writev(
static_cast<int>(context->descriptor), vectors, 2);
1193 context->noProgressError = thread->
getErrno();
1195 context->file->setMakeProgress(
true);
1197 context->partialResult = posix_writev(
static_cast<int>(context->descriptor), vectors, 2);
1198 context->partialError = thread->
getErrno();
1199 context->returned += 1;
1203bool vectorSignalProgress(
Process* kernelProcess) {
1204 constexpr size_t Descriptor = 95;
1208 process->setSubsystem(subsystem);
1209 InterruptingVectorFile file;
1214 VectorSignalContext context(Descriptor, &file);
1215 Thread*
worker =
new Thread(process, vectorSignalWorker, &context,
nullptr,
false,
true,
true);
1216 worker->setName(
"hosted vector signal progress");
1217 const bool started =
worker->start();
1218 const bool joined = started &&
worker->joinForCompletion();
1223 bool passed = started && joined && context.returned == 1 && context.noProgressResult == -1 &&
1224 context.noProgressError == Error::Interrupted && context.partialResult == 3 &&
1225 context.partialError == PreservedErrno && file.calls() == 2 &&
1227 passed = closeDescriptor(subsystem, Descriptor) && passed;
1228 description.
reset();
1233 "HOSTED-SYSCALL-TEST: FAIL vector-signal-progress: "
1234 "writev did not distinguish interruption before and after progress");
1238 NOTICE(
"HOSTED-SYSCALL-TEST: PASS vector-signal-progress");
1242struct LargeVectorPipeContext {
1243 LargeVectorPipeContext(
size_t descriptor,
bool largeFirst)
1244 : descriptor(descriptor), largeFirst(largeFirst), result(-2), error(0), returned(0) {}
1253int largeVectorPipeWriter(
void* parameter) {
1254 LargeVectorPipeContext* context =
reinterpret_cast<LargeVectorPipeContext*
>(parameter);
1255 char large[PIPE_BUF_MAX];
1256 char small = context->largeFirst ?
'b' :
'a';
1257 ByteSet(large, context->largeFirst ?
'a' :
'b', sizeof(large));
1258 struct iovec vectors[2] = {
1259 {context->largeFirst ?
static_cast<void*
>(large) : static_cast<void*>(&small),
1260 context->largeFirst ? sizeof(large) : 1},
1261 {context->largeFirst ?
static_cast<void*
>(&small) : static_cast<void*>(large),
1262 context->largeFirst ? 1 : sizeof(large)}};
1266 context->result = posix_writev(
static_cast<int>(context->descriptor), vectors, 2);
1267 context->error = thread->
getErrno();
1268 context->returned += 1;
1273 size_t descriptor,
bool largeFirst) {
1274 char fill[PIPE_BUF_MAX];
1275 ByteSet(fill,
'f',
sizeof(fill));
1276 if (writer->
write(
sizeof(fill),
reinterpret_cast<uintptr_t
>(fill),
true) !=
sizeof(fill)) {
1281 if (
reader->read(1,
reinterpret_cast<uintptr_t
>(&
byte),
true) != 1 ||
byte !=
'f') {
1285 LargeVectorPipeContext context(descriptor, largeFirst);
1286 Thread*
worker =
new Thread(process, largeVectorPipeWriter, &context,
nullptr,
false,
true,
true);
1287 worker->setName(largeFirst ?
"hosted large-first vector pipe" :
"hosted small-first vector pipe");
1288 const bool started =
worker->start();
1289 const bool joined = started &&
worker->joinForCompletion();
1294 char drain[PIPE_BUF_MAX];
1295 const bool drained = joined &&
reader->read(
sizeof(drain),
reinterpret_cast<uintptr_t
>(drain),
1296 true) ==
sizeof(drain);
1297 return started && joined && drained && context.returned == 1 && context.result == 1 &&
1298 context.error == PreservedErrno && drain[PIPE_BUF_MAX - 1] ==
'a';
1301bool vectorPipeBoundaryIndependentPartial(
Process* kernelProcess) {
1302 constexpr size_t ReadDescriptor = 96;
1303 constexpr size_t WriteDescriptor = 97;
1307 process->setSubsystem(subsystem);
1314 bool passed = runLargeVectorPipeCase(process,
reader, writer, WriteDescriptor,
true) &&
1315 runLargeVectorPipeCase(process,
reader, writer, WriteDescriptor,
false);
1316 passed = closeDescriptor(subsystem, WriteDescriptor) &&
1317 closeDescriptor(subsystem, ReadDescriptor) && passed;
1322 "HOSTED-SYSCALL-TEST: FAIL vector-pipe-boundary-independent-partial: "
1323 "large nonblocking writev behavior depended on an iovec boundary");
1327 NOTICE(
"HOSTED-SYSCALL-TEST: PASS vector-pipe-boundary-independent-partial");
1332File* g_VectorSigpipeTarget =
nullptr;
1334void vectorSigpipeHandler(
size_t) {
1336 g_VectorSigpipeHandlerCalls += 1;
1339struct VectorSigpipeContext {
1340 explicit VectorSigpipeContext(
size_t descriptor)
1341 : descriptor(descriptor), result(-2), error(0), returned(0) {}
1349int vectorSigpipeWriter(
void* parameter) {
1350 VectorSigpipeContext* context =
reinterpret_cast<VectorSigpipeContext*
>(parameter);
1353 struct iovec vectors[2] = {{&first, 1}, {&second, 1}};
1356 context->result = posix_writev(
static_cast<int>(context->descriptor), vectors, 2);
1357 context->error = thread->
getErrno();
1358 context->returned += 1;
1362bool vectorSigpipeAfterWriteGuard(
Process* kernelProcess) {
1363 constexpr size_t WriteDescriptor = 98;
1367 process->setSubsystem(subsystem);
1373 handler->
pEvent =
new SignalEvent(
reinterpret_cast<uintptr_t
>(&vectorSigpipeHandler), SIGPIPE);
1376 g_VectorSigpipeHandlerCalls = 0;
1377 g_VectorSigpipeTarget = pipe;
1378 VectorSigpipeContext context(WriteDescriptor);
1379 Thread*
worker =
new Thread(process, vectorSigpipeWriter, &context,
nullptr,
false,
true,
true);
1380 worker->setName(
"hosted vector SIGPIPE guard release");
1381 const bool started =
worker->start();
1382 const bool joined = started &&
worker->joinForCompletion();
1386 g_VectorSigpipeTarget =
nullptr;
1388 bool passed = started && joined && context.returned == 1 && context.result == -1 &&
1389 context.error == Error::BrokenPipe && g_VectorSigpipeHandlerCalls == 1;
1390 passed = closeDescriptor(subsystem, WriteDescriptor) && passed;
1395 "HOSTED-SYSCALL-TEST: FAIL vector-sigpipe-after-write-guard: "
1396 "SIGPIPE ran before vector write serialization was released");
1400 NOTICE(
"HOSTED-SYSCALL-TEST: PASS vector-sigpipe-after-write-guard");
1405bool runHostedVectorIoRegressions(
Process* process) {
1406 return vectorWriteBounceAndLifetime(process) && vectorReadBounceAndScatter(process) &&
1407 diskVectorReadBatching(process) && vectorUsercopyFaultProgress(process) &&
1408 vectorPipePayloadPrevalidation(process) && vectorSignalProgress(process) &&
1409 vectorPipeBoundaryIndependentPartial(process) && vectorSigpipeAfterWriteGuard(process);
Memory-mapped file interface.
virtual void unpin(uint64_t location)=0
virtual MUST_USE_RESULT bool pin(uint64_t location)=0
Pins a cache page.
OpenFileDescriptionLease acquireOpenFileDescription() const
size_t fd
Descriptor number.
uint64_t write(uint64_t size, uintptr_t buffer, bool canBlock=true)
uint64_t getOffset() const
virtual uintptr_t readBlock(uint64_t location)
virtual uint64_t readBytewise(uint64_t location, uint64_t size, uintptr_t buffer, bool bCanBlock=true)
virtual bool isBytewise() const
virtual uint64_t writeBytewise(uint64_t location, uint64_t size, uintptr_t buffer, bool bCanBlock=true)
virtual MUST_USE_RESULT bool pinBlock(uint64_t location)
virtual bool removeNode(File *parent, const String &filename, File *file)=0
virtual bool initialise(Disk *pDisk)=0
virtual const String & getVolumeLabel() const =0
bool createSymlink(const StringView &path, const String &value, File *pStartNode=0)
bool createDirectory(const StringView &path, uint32_t mask, File *pStartNode=0)
virtual File * getRoot() const =0
bool createFile(const StringView &path, uint32_t mask, File *pStartNode=0)
MemoryMappedObject * mapAnon(uintptr_t &address, size_t length, MemoryMappedObject::Permissions perms)
size_t remove(uintptr_t base, size_t length)
static MemoryMapManager & instance()
static constexpr size_t getPageSize() PURE
bool acquireFileDescriptor(size_t fd, DescriptorLease &descriptor)
bool closeFileDescriptor(size_t fd, const DescriptorLease &descriptor)
void addFileDescriptor(size_t fd, FileDescriptor *pFd)
void setSignalHandler(size_t sig, SignalHandler *handler)
static ProcessorInformation & information()
static Scheduler & instance()
void setErrno(size_t err)
SignalEvent * pEvent
Event for the signal handler.