8#include "pedigree/kernel/Atomic.h"
9#include "pedigree/kernel/Log.h"
10#include "pedigree/kernel/errors.h"
11#include "pedigree/kernel/process/Process.h"
12#include "pedigree/kernel/process/Scheduler.h"
13#include "pedigree/kernel/process/Semaphore.h"
14#include "pedigree/kernel/process/Thread.h"
15#include "pedigree/kernel/processor/PhysicalMemoryManager.h"
16#include "pedigree/kernel/processor/Processor.h"
17#include "pedigree/kernel/processor/VirtualAddressSpace.h"
18#include "pedigree/kernel/utilities/utility.h"
23#include "modules/subsys/posix/FileDescriptor.h"
24#include "modules/subsys/posix/PosixSubsystem.h"
25#include "modules/subsys/posix/file-syscalls.h"
26#include "modules/system/vfs/File.h"
28#include "modules/system/vfs/Pipe.h"
31constexpr size_t BounceCapacity = PIPE_BUF_MAX + 1;
32constexpr size_t ChunkedWriteLength = BounceCapacity * 2 + 17;
33constexpr int PreservedErrno = 123;
41class ScalarWriteProbeFile final :
public File {
43 ScalarWriteProbeFile()
44 :
File(
String(
"scalar-write-probe"), 0, 0, 0, 1, nullptr, 0, nullptr),
45 m_FirstWriteEntered(0, false),
46 m_ReleaseFirstWrite(0, false),
47 m_UserSource(nullptr),
50 m_Offsets{0, 0, 0, 0},
52 m_FirstValues{0, 0, 0, 0} {}
54 void setUserSource(
char* source) {
55 m_UserSource = source;
58 bool waitForFirstWrite() {
59 return m_FirstWriteEntered.acquireForCompletion();
62 void releaseFirstWrite() {
63 m_ReleaseFirstWrite.release();
66 size_t writeCount()
const {
70 uint64_t offset(
size_t index)
const {
71 return m_Offsets[index];
74 size_t size(
size_t index)
const {
75 return m_Sizes[index];
78 char firstValue(
size_t index)
const {
79 return m_FirstValues[index];
82 bool sawRawPointer()
const {
91 uint64_t
writeBytewise(uint64_t location, uint64_t size, uintptr_t buffer,
bool)
override {
92 const size_t slot = (m_WriteCount += 1) - 1;
94 m_Offsets[slot] = location;
99 m_RawPointer = buffer ==
reinterpret_cast<uintptr_t
>(m_UserSource);
100 ByteSet(m_UserSource,
'z', ChunkedWriteLength);
103 if (slot < 4 && size) {
104 m_FirstValues[slot] = *
reinterpret_cast<const char*
>(buffer);
108 m_FirstWriteEntered.release();
109 if (!m_ReleaseFirstWrite.acquireForCompletion()) {
114 if (location + size > getSize()) {
115 setSize(location + size);
126 uint64_t m_Offsets[4];
128 char m_FirstValues[4];
131class ScalarReplacementFile final :
public File {
133 ScalarReplacementFile()
134 :
File(
String(
"scalar-replacement"), 0, 0, 0, 2, nullptr, 0, nullptr), m_Writes(0) {}
136 size_t writes()
const {
145 uint64_t
writeBytewise(uint64_t, uint64_t size, uintptr_t,
bool)
override {
154struct ScalarWriteContext {
155 ScalarWriteContext(
size_t descriptor, ScalarWriteProbeFile* file)
156 : descriptor(descriptor), file(file), entered(0), result(-2), error(0), returned(0) {}
159 ScalarWriteProbeFile* file;
166int chunkedScalarWrite(
void* parameter) {
167 ScalarWriteContext* context =
reinterpret_cast<ScalarWriteContext*
>(parameter);
168 char payload[ChunkedWriteLength];
169 ByteSet(payload,
'a',
sizeof(payload));
170 context->file->setUserSource(payload);
171 context->entered += 1;
176 posix_write(
static_cast<int>(context->descriptor), payload,
sizeof(payload),
false);
177 context->error = thread->
getErrno();
178 context->returned += 1;
182struct AliasWriteContext {
183 explicit AliasWriteContext(
size_t descriptor)
184 : descriptor(descriptor), entered(0), result(-2), error(0), returned(0) {}
193int scalarAliasWrite(
void* parameter) {
194 AliasWriteContext* context =
reinterpret_cast<AliasWriteContext*
>(parameter);
196 context->entered += 1;
199 context->result = posix_write(
static_cast<int>(context->descriptor), &value, 1,
false);
200 context->error = thread->
getErrno();
201 context->returned += 1;
205bool scalarWriteBounceAndLifetime(
Process* kernelProcess) {
206 constexpr size_t SourceDescriptor = 82;
207 constexpr size_t AliasDescriptor = 83;
211 process->setSubsystem(subsystem);
213 ScalarWriteProbeFile sourceFile;
214 ScalarReplacementFile replacementFile;
217 alias->
fd = AliasDescriptor;
222 ScalarWriteContext sourceContext(SourceDescriptor, &sourceFile);
223 AliasWriteContext aliasContext(AliasDescriptor);
225 new Thread(process, chunkedScalarWrite, &sourceContext,
nullptr,
false,
true,
true);
227 new Thread(process, scalarAliasWrite, &aliasContext,
nullptr,
false,
true,
true);
228 sourceWorker->setName(
"hosted scalar bounce writer");
229 aliasWorker->setName(
"hosted scalar bounce alias");
231 const bool sourceStarted = sourceWorker->
start();
232 const bool firstEntered = sourceStarted && sourceFile.waitForFirstWrite();
233 const bool aliasStarted = firstEntered && aliasWorker->
start();
234 while (aliasStarted && !aliasContext.entered) {
237 for (
size_t attempt = 0; attempt < 32 && aliasStarted && !aliasContext.returned; ++attempt) {
240 const bool aliasWasSerialized = aliasStarted && !aliasContext.returned;
242 const bool sourceClosed = firstEntered && closeDescriptor(subsystem, SourceDescriptor);
244 SourceDescriptor,
new FileDescriptor(&replacementFile, 0, SourceDescriptor, 0, O_WRONLY));
246 sourceFile.releaseFirstWrite();
249 if (!sourceStarted) {
256 bool passed = sourceStarted && firstEntered && aliasStarted && aliasWasSerialized &&
257 sourceClosed && sourceJoined && aliasJoined && sourceContext.returned == 1 &&
258 sourceContext.result ==
static_cast<int>(ChunkedWriteLength) &&
259 sourceContext.error == PreservedErrno && aliasContext.returned == 1 &&
260 aliasContext.result == 1 && aliasContext.error == PreservedErrno &&
261 !sourceFile.sawRawPointer() && sourceFile.writeCount() == 4 &&
262 sourceFile.offset(0) == 0 && sourceFile.size(0) == BounceCapacity &&
263 sourceFile.firstValue(0) ==
'a' && sourceFile.offset(1) == BounceCapacity &&
264 sourceFile.size(1) == BounceCapacity && sourceFile.firstValue(1) ==
'z' &&
265 sourceFile.offset(2) == BounceCapacity * 2 && sourceFile.size(2) == 17 &&
266 sourceFile.firstValue(2) ==
'z' && sourceFile.offset(3) == ChunkedWriteLength &&
267 sourceFile.size(3) == 1 && sourceFile.firstValue(3) ==
'q' &&
268 replacementFile.writes() == 0 && alias->
getOffset() == ChunkedWriteLength + 1;
270 const bool aliasClosed = closeDescriptor(subsystem, AliasDescriptor);
271 const bool replacementClosed = closeDescriptor(subsystem, SourceDescriptor);
272 passed = passed && aliasClosed && replacementClosed;
278 "HOSTED-SYSCALL-TEST: FAIL scalar-write-bounce-lifetime: "
279 "the scalar write exposed a user pointer, lost its descriptor/OFD, or interleaved chunks");
283 NOTICE(
"HOSTED-SYSCALL-TEST: PASS scalar-write-bounce-lifetime");
287class FaultingScalarFile final :
public File {
289 explicit FaultingScalarFile(
bool reading)
290 :
File(
String(reading ?
"scalar-read-fault" :
"scalar-write-fault"), 0, 0, 0, reading ? 3 : 4,
291 nullptr, 0, nullptr),
299 void configure(uintptr_t userBase,
size_t mappingLength,
size_t pageSize) {
300 m_UserBase = userBase;
301 m_MappingLength = mappingLength;
302 m_PageSize = pageSize;
305 size_t calls()
const {
309 bool sawRawPointer()
const {
313 char firstValue()
const {
322 uint64_t
readBytewise(uint64_t, uint64_t size, uintptr_t buffer,
bool)
override {
323 const size_t slot = (m_Calls += 1) - 1;
324 const bool raw = buffer >= m_UserBase && buffer < m_UserBase + m_MappingLength;
325 m_RawPointer = m_RawPointer || raw;
329 m_UserBase + m_PageSize, m_MappingLength - m_PageSize, MemoryMappedObject::Read);
332 ByteSet(
reinterpret_cast<void*
>(buffer),
'r', size);
337 uint64_t
writeBytewise(uint64_t, uint64_t size, uintptr_t buffer,
bool)
override {
338 const size_t slot = (m_Calls += 1) - 1;
339 const bool raw = buffer >= m_UserBase && buffer < m_UserBase + m_MappingLength;
340 m_RawPointer = m_RawPointer || raw;
342 m_FirstValue = *
reinterpret_cast<const char*
>(buffer);
347 m_UserBase + m_PageSize, m_MappingLength - m_PageSize, MemoryMappedObject::None);
353 uintptr_t m_UserBase;
354 size_t m_MappingLength;
361struct ScalarFaultContext {
362 ScalarFaultContext(
Process* process,
size_t writeFd,
size_t readFd, FaultingScalarFile* writeFile,
363 FaultingScalarFile* readFile)
367 writeFile(writeFile),
372 firstWriteFaultResult(-2),
373 firstWriteFaultError(0),
376 firstReadFaultResult(-2),
377 firstReadFaultError(0),
383 FaultingScalarFile* writeFile;
384 FaultingScalarFile* readFile;
388 int firstWriteFaultResult;
389 int firstWriteFaultError;
392 int firstReadFaultResult;
393 int firstReadFaultError;
397bool allocateUserMapping(
Process* process,
size_t length, uintptr_t& address) {
399 if (!process->allocateUserRange(Process::UserRegion::Normal, length, address)) {
403 uintptr_t mappedAddress = address;
405 mappedAddress, length, MemoryMappedObject::Read | MemoryMappedObject::Write);
406 if (!mapping || mappedAddress != address) {
408 process->freeUserRange(Process::UserRegion::Normal, address, length);
415int scalarFaultWorker(
void* parameter) {
416 ScalarFaultContext* context =
reinterpret_cast<ScalarFaultContext*
>(parameter);
419 const size_t mappingLength = pageSize * 3;
420 const size_t transferLength = BounceCapacity * 2;
422 uintptr_t writeAddress = 0;
423 if (!allocateUserMapping(context->process, mappingLength, writeAddress)) {
424 context->returned += 1;
427 ByteSet(
reinterpret_cast<void*
>(writeAddress),
'w', mappingLength);
428 context->writeFile->configure(writeAddress, mappingLength, pageSize);
430 context->writeResult = posix_write(
static_cast<int>(context->writeFd),
431 reinterpret_cast<char*
>(writeAddress), transferLength,
false);
432 context->writeError = thread->
getErrno();
436 context->firstWriteFaultResult = posix_write(
static_cast<int>(context->writeFd),
437 reinterpret_cast<char*
>(kernelStart), 1,
false);
438 context->firstWriteFaultError = thread->
getErrno();
440 context->process->freeUserRange(Process::UserRegion::Normal, writeAddress, mappingLength);
442 uintptr_t readAddress = 0;
443 if (!allocateUserMapping(context->process, mappingLength, readAddress)) {
444 context->returned += 1;
447 ByteSet(
reinterpret_cast<void*
>(readAddress), 0, mappingLength);
448 context->readFile->configure(readAddress, mappingLength, pageSize);
450 context->readResult = posix_read(
static_cast<int>(context->readFd),
451 reinterpret_cast<char*
>(readAddress), transferLength);
452 context->readError = thread->
getErrno();
455 context->firstReadFaultResult =
456 posix_read(
static_cast<int>(context->readFd),
reinterpret_cast<char*
>(kernelStart), 1);
457 context->firstReadFaultError = thread->
getErrno();
459 context->process->freeUserRange(Process::UserRegion::Normal, readAddress, mappingLength);
461 context->setup =
true;
462 context->returned += 1;
466bool scalarUsercopyFaultProgress(
Process* kernelProcess) {
467 constexpr size_t WriteDescriptor = 84;
468 constexpr size_t ReadDescriptor = 85;
472 process->setSubsystem(subsystem);
473 FaultingScalarFile writeFile(
false);
474 FaultingScalarFile readFile(
true);
482 ScalarFaultContext context(process, WriteDescriptor, ReadDescriptor, &writeFile, &readFile);
483 Thread*
worker =
new Thread(process, scalarFaultWorker, &context,
nullptr,
false,
true,
true);
484 worker->setName(
"hosted scalar usercopy fault progress");
485 const bool started =
worker->start();
486 const bool joined = started &&
worker->joinForCompletion();
491 bool passed = started && joined && context.returned == 1 && context.setup &&
492 context.writeResult ==
static_cast<int>(BounceCapacity) &&
493 context.writeError == PreservedErrno && context.firstWriteFaultResult == -1 &&
494 context.firstWriteFaultError == Error::BadAddress && writeFile.calls() == 1 &&
495 !writeFile.sawRawPointer() && writeFile.firstValue() ==
'w' &&
497 context.readResult ==
static_cast<int>(BounceCapacity) &&
498 context.readError == PreservedErrno && context.firstReadFaultResult == -1 &&
499 context.firstReadFaultError == Error::BadAddress && readFile.calls() == 2 &&
500 !readFile.sawRawPointer() &&
reader->getOffset() == BounceCapacity;
502 passed = closeDescriptor(subsystem, WriteDescriptor) &&
503 closeDescriptor(subsystem, ReadDescriptor) && passed;
504 writeDescription.
reset();
505 readDescription.
reset();
510 "HOSTED-SYSCALL-TEST: FAIL scalar-usercopy-fault-progress: "
511 "first/later EFAULT handling exposed pointers or lost partial offset semantics");
515 NOTICE(
"HOSTED-SYSCALL-TEST: PASS scalar-usercopy-fault-progress");
519class SnapshotPipe final :
public Pipe {
522 :
Pipe(
String(
""), 0, 0, 0, 0, nullptr, 0, nullptr, true),
524 m_UserSource(nullptr),
529 void arm(
char* source) {
530 m_UserSource = source;
534 bool waitUntilEntered() {
535 return m_Entered.acquireForCompletion();
538 bool sawRawPointer()
const {
542 char firstValue()
const {
547 uint64_t
writeBytewise(uint64_t location, uint64_t size, uintptr_t buffer,
548 bool canBlock)
override {
551 m_RawPointer = buffer ==
reinterpret_cast<uintptr_t
>(m_UserSource);
552 ByteSet(m_UserSource,
'z', PIPE_BUF_MAX);
554 m_FirstValue = *
reinterpret_cast<const char*
>(buffer);
569struct ScalarPipeContext {
570 ScalarPipeContext(
size_t writeFd, SnapshotPipe* pipe)
573 atomicComplete(0, false),
574 runLargeWrite(0, false),
590int scalarPipeWriter(
void* parameter) {
591 ScalarPipeContext* context =
reinterpret_cast<ScalarPipeContext*
>(parameter);
592 char atomicPayload[PIPE_BUF_MAX];
593 ByteSet(atomicPayload,
'a',
sizeof(atomicPayload));
594 context->pipe->arm(atomicPayload);
595 context->atomicResult =
596 posix_write(
static_cast<int>(context->writeFd), atomicPayload,
sizeof(atomicPayload),
false);
597 context->atomicComplete.release();
599 if (!context->runLargeWrite.acquireForCompletion()) {
600 context->returned += 1;
604 char largePayload[PIPE_BUF_MAX + 1];
605 ByteSet(largePayload,
'l',
sizeof(largePayload));
608 context->largeResult =
609 posix_write(
static_cast<int>(context->writeFd), largePayload,
sizeof(largePayload),
false);
610 context->largeError = thread->
getErrno();
611 context->returned += 1;
615bool scalarPipeSnapshotAndLargePartial(
Process* kernelProcess) {
616 constexpr size_t ReadDescriptor = 86;
617 constexpr size_t WriteDescriptor = 87;
621 process->setSubsystem(subsystem);
622 SnapshotPipe* pipe =
new SnapshotPipe;
628 char fill[PIPE_BUF_MAX];
629 ByteSet(fill,
'f',
sizeof(fill));
631 writer->
write(
sizeof(fill),
reinterpret_cast<uintptr_t
>(fill),
true) ==
sizeof(fill);
633 ScalarPipeContext context(WriteDescriptor, pipe);
634 Thread*
worker =
new Thread(process, scalarPipeWriter, &context,
nullptr,
false,
true,
true);
635 worker->setName(
"hosted scalar pipe bounce writer");
636 const bool started = filled &&
worker->start();
637 const bool entered = started && pipe->waitUntilEntered();
639 char drain[PIPE_BUF_MAX];
640 const bool initialDrained =
642 reader->read(
sizeof(drain),
reinterpret_cast<uintptr_t
>(drain),
true) ==
sizeof(drain);
643 const bool atomicCompleted = initialDrained && context.atomicComplete.acquireForCompletion();
644 const bool atomicDrained =
646 reader->read(
sizeof(drain),
reinterpret_cast<uintptr_t
>(drain),
true) ==
sizeof(drain);
647 bool atomicContents = atomicDrained;
648 for (
size_t i = 0; i <
sizeof(drain) && atomicContents; ++i) {
649 atomicContents = drain[i] ==
'a';
652 const bool refilled =
654 writer->
write(
sizeof(fill),
reinterpret_cast<uintptr_t
>(fill),
true) ==
sizeof(fill);
656 const bool oneByteFreed =
657 refilled &&
reader->read(1,
reinterpret_cast<uintptr_t
>(&
byte),
true) == 1;
659 context.runLargeWrite.release();
661 const bool joined = started &&
worker->joinForCompletion();
665 const bool finalDrained =
667 reader->read(
sizeof(drain),
reinterpret_cast<uintptr_t
>(drain),
true) ==
sizeof(drain);
669 bool passed = started && entered && initialDrained && atomicCompleted && atomicDrained &&
670 atomicContents && refilled && oneByteFreed && joined && finalDrained &&
671 context.returned == 1 && context.atomicResult == PIPE_BUF_MAX &&
672 !pipe->sawRawPointer() && pipe->firstValue() ==
'a' && context.largeResult == 1 &&
673 context.largeError == PreservedErrno && drain[PIPE_BUF_MAX - 1] ==
'l';
675 passed = closeDescriptor(subsystem, WriteDescriptor) &&
676 closeDescriptor(subsystem, ReadDescriptor) && passed;
681 "HOSTED-SYSCALL-TEST: FAIL scalar-pipe-bounce: "
682 "PIPE_BUF snapshot atomicity or the 4097-byte nonblocking partial write regressed");
686 NOTICE(
"HOSTED-SYSCALL-TEST: PASS scalar-pipe-bounce");
691File* g_ScalarSigpipeTarget =
nullptr;
693void scalarSigpipeHandler(
size_t) {
695 g_ScalarSigpipeHandlerCalls += 1;
698struct ScalarSigpipeContext {
699 explicit ScalarSigpipeContext(
size_t descriptor)
700 : descriptor(descriptor), result(-2), error(0), returned(0) {}
708int scalarSigpipeWriter(
void* parameter) {
709 ScalarSigpipeContext* context =
reinterpret_cast<ScalarSigpipeContext*
>(parameter);
714 posix_write(
static_cast<int>(context->descriptor), &value,
sizeof(value),
false);
715 context->error = thread->
getErrno();
716 context->returned += 1;
720bool scalarSigpipeAfterWriteGuard(
Process* kernelProcess) {
721 constexpr size_t WriteDescriptor = 88;
725 process->setSubsystem(subsystem);
731 handler->
pEvent =
new SignalEvent(
reinterpret_cast<uintptr_t
>(&scalarSigpipeHandler), SIGPIPE);
734 g_ScalarSigpipeHandlerCalls = 0;
735 g_ScalarSigpipeTarget = pipe;
736 ScalarSigpipeContext context(WriteDescriptor);
737 Thread*
worker =
new Thread(process, scalarSigpipeWriter, &context,
nullptr,
false,
true,
true);
738 worker->setName(
"hosted scalar SIGPIPE guard release");
739 const bool started =
worker->start();
740 const bool joined = started &&
worker->joinForCompletion();
744 g_ScalarSigpipeTarget =
nullptr;
746 bool passed = started && joined && context.returned == 1 && context.result == -1 &&
747 context.error == Error::BrokenPipe && g_ScalarSigpipeHandlerCalls == 1;
748 passed = closeDescriptor(subsystem, WriteDescriptor) && passed;
753 "HOSTED-SYSCALL-TEST: FAIL scalar-sigpipe-after-write-guard: "
754 "SIGPIPE ran before scalar write serialization was released");
758 NOTICE(
"HOSTED-SYSCALL-TEST: PASS scalar-sigpipe-after-write-guard");
763bool runHostedScalarIoRegressions(
Process* process) {
764 return scalarWriteBounceAndLifetime(process) && scalarUsercopyFaultProgress(process) &&
765 scalarPipeSnapshotAndLargePartial(process) && scalarSigpipeAfterWriteGuard(process);
Memory-mapped file interface.
OpenFileDescriptionLease acquireOpenFileDescription() const
void addStatusFlag(int newFlag)
Helper to add a single flag to the status flags.
size_t fd
Descriptor number.
uint64_t write(uint64_t size, uintptr_t buffer, bool canBlock=true)
uint64_t getOffset() const
virtual uint64_t readBytewise(uint64_t location, uint64_t size, uintptr_t buffer, bool bCanBlock=true)
virtual bool isBytewise() const
virtual uint64_t writeBytewise(uint64_t location, uint64_t size, uintptr_t buffer, bool bCanBlock=true)
MemoryMappedObject * mapAnon(uintptr_t &address, size_t length, MemoryMappedObject::Permissions perms)
size_t remove(uintptr_t base, size_t length)
static MemoryMapManager & instance()
static constexpr size_t getPageSize() PURE
virtual uint64_t writeBytewise(uint64_t location, uint64_t size, uintptr_t buffer, bool bCanBlock=true)
bool acquireFileDescriptor(size_t fd, DescriptorLease &descriptor)
bool closeFileDescriptor(size_t fd, const DescriptorLease &descriptor)
void addFileDescriptor(size_t fd, FileDescriptor *pFd)
void setSignalHandler(size_t sig, SignalHandler *handler)
static ProcessorInformation & information()
static Scheduler & instance()
void setErrno(size_t err)
SignalEvent * pEvent
Event for the signal handler.