8#include "pedigree/kernel/Log.h"
9#include "pedigree/kernel/process/Process.h"
10#include "pedigree/kernel/process/Scheduler.h"
11#include "pedigree/kernel/process/Thread.h"
12#include "pedigree/kernel/process/Uninterruptible.h"
13#include "pedigree/kernel/processor/MemoryRegion.h"
14#include "pedigree/kernel/processor/PhysicalMemoryManager.h"
15#include "pedigree/kernel/processor/Processor.h"
16#include "pedigree/kernel/processor/VirtualAddressSpace.h"
17#include "pedigree/kernel/syscallError.h"
18#include "pedigree/kernel/utilities/utility.h"
20#include "modules/subsys/posix/file-syscalls.h"
21#include "modules/system/vfs/File.h"
26bool check(
bool condition,
const char* detail) {
28 ERROR(
"VM-OWNERSHIP-TEST: FAIL " << detail);
33#if HOSTED && PEDIGREE_HOSTED_SMOKE_TESTS
34bool protectedClone(
bool inaccessible) {
40 void* address =
reinterpret_cast<void*
>(source->
getDynamicStart() + 64 * pageSize);
41 const physical_uintptr_t physical = memory.
allocatePage();
47 return check(
false,
"private mapping setup");
50 volatile uint8_t* bytes =
reinterpret_cast<volatile uint8_t*
>(address);
52 bytes[pageSize - 1] = 0x7A;
57 physical_uintptr_t parentPhysical = 0;
58 physical_uintptr_t childPhysical = 0;
59 size_t parentFlags = 0;
60 size_t childFlags = 0;
61 source->
getMapping(address, parentPhysical, parentFlags);
62 child->
getMapping(address, childPhysical, childFlags);
64 check(source->
isMapped(address) && child->
isMapped(address) && parentPhysical == physical &&
71 "protected clone changed ownership or permissions");
75 PhysicalMemoryManager::pageReferenceCountForTest(physical) == 2,
76 "copy-on-write bypassed read-only protection");
81 passed &= check(parentResolved,
"parent write upgrade could not resolve");
83 passed &= check(bytes[0] == 0x31 && bytes[pageSize - 1] == 0x7A,
84 "restoring parent access lost contents");
87 source->
getMapping(address, parentPhysical, parentFlags);
93 passed &= check(childResolved,
"child write upgrade could not resolve");
95 passed &= check(bytes[0] == 0x31 && bytes[pageSize - 1] == 0x7A,
96 "parent write changed protected child");
99 child->
getMapping(address, childPhysical, childFlags);
101 if (parentResolved) {
102 passed &= check(bytes[0] == 0xA1,
"child write changed parent");
106 delete sourceProcess;
107 passed &= check(PhysicalMemoryManager::pageReferenceCountForTest(physical) == 0 &&
108 PhysicalMemoryManager::pageReferenceCountForTest(parentPhysical) == 0 &&
109 PhysicalMemoryManager::pageReferenceCountForTest(childPhysical) == 0,
110 "protected clones leaked a physical owner");
114bool borrowedClones() {
120 void* address =
reinterpret_cast<void*
>(source->
getDynamicStart() + 64 * pageSize);
121 const physical_uintptr_t physical = memory.
allocatePage();
122 if (!physical || !source->
map(physical, address,
128 delete sourceProcess;
129 return check(
false,
"borrowed mapping setup");
131 const size_t references = PhysicalMemoryManager::pageReferenceCountForTest(physical);
135 physical_uintptr_t clonePhysical = 0;
136 size_t cloneFlags = 0;
141 PhysicalMemoryManager::pageReferenceCountForTest(physical) == references,
142 "borrowed clones acquired physical ownership");
147 "conditional detach changed a mismatched mapping");
152 PhysicalMemoryManager::pageReferenceCountForTest(physical) == references,
153 "foreign protected detach changed backing ownership");
157 delete sourceProcess;
158 passed &= check(PhysicalMemoryManager::pageReferenceCountForTest(physical) == references,
159 "borrowed teardown released backing ownership");
161 passed &= check(PhysicalMemoryManager::pageReferenceCountForTest(physical) == 0,
162 "backing owner could not release borrowed page");
167class ResizeProbeFile final :
public File {
169 explicit ResizeProbeFile(
size_t pages = 2)
170 :
File(
String(
"mapped-resize-probe"), 0, 0, 0, 1, nullptr,
172 storage(
"Mapped Resize Probe"),
174 rejectWritableMapping(false),
187 ByteSet(storage.virtualAddress(), 0x49, getSize());
192 bool sync(
size_t,
bool)
override {
197 return __atomic_load_n(&physicalPageLoans(), __ATOMIC_ACQUIRE);
200 if (rejectWritableMapping) {
201 SYSCALL_ERROR(OutOfMemory);
209 bool rejectWritableMapping;
210 size_t preparedLoans;
211 size_t committedLoans;
212 size_t shrinkCommits;
217 uintptr_t
readBlock(uint64_t location)
override {
218 return reinterpret_cast<uintptr_t
>(storage.
virtualAddress()) + location;
226 ShrinkPlan(ResizeProbeFile& file,
size_t size) : file(file), size(size) {}
227 void commit()
override {
228 file.committedLoans = file.loans();
229 ++file.shrinkCommits;
232 ResizeProbeFile& file;
235 bool prepareShrink(
const ShrinkContext& context,
237 preparedLoans = loans();
239 SYSCALL_ERROR(IoError);
244 SYSCALL_ERROR(OutOfMemory);
249 bool resizeFile(
size_t size)
override {
251 SYSCALL_ERROR(IoError);
259#if HOSTED && PEDIGREE_HOSTED_SMOKE_TESTS
260int resizeFailureWorker(
void* parameter) {
261 bool& passed = *
static_cast<bool*
>(parameter);
263 ResizeProbeFile file;
264 if (!file.initialise()) {
268 uintptr_t shared = 0;
269 uintptr_t privateAddress = 0;
270 auto permissions = MemoryMappedObject::Read | MemoryMappedObject::Write;
272 manager.
mapFile(&file, shared, 2 * pageSize, MemoryMappedObject::Read, 0,
false);
274 manager.
mapFile(&file, privateAddress, 2 * pageSize, permissions);
275 if (!sharedObject || !privateObject || !manager.faultIn(shared,
false) ||
276 !manager.faultIn(shared + pageSize,
false) ||
277 !manager.faultIn(privateAddress + pageSize,
true)) {
283 volatile uint8_t* privateByte =
reinterpret_cast<volatile uint8_t*
>(privateAddress + pageSize);
286 physical_uintptr_t owned = 0;
288 file.rejectWritableMapping =
true;
290 passed = check(!manager.setPermissions(shared, 2 * pageSize, permissions, &status) &&
291 status == MemoryMapManager::ProtectStatus::NoMemory,
292 "writable protection ignored failed backing preparation");
293 space.
getMapping(
reinterpret_cast<void*
>(shared + pageSize), owned, flags);
295 "failed backing preparation published write permission");
296 file.rejectWritableMapping =
false;
297 passed &= check(manager.setPermissions(shared, 2 * pageSize, permissions) != 0,
298 "writable protection could not retry backing preparation");
299 space.
getMapping(
const_cast<uint8_t*
>(privateByte), owned, flags);
300 const size_t privateFlags = flags;
301 physical_uintptr_t borrowed = 0;
302 size_t borrowedFlags = 0;
303 physical_uintptr_t prefix = 0;
304 size_t prefixFlags = 0;
305 space.
getMapping(
reinterpret_cast<void*
>(shared + pageSize), borrowed, borrowedFlags);
306 space.
getMapping(
reinterpret_cast<void*
>(shared), prefix, prefixFlags);
307 const size_t originalLoans = file.loans();
308 const size_t privateReferences = PhysicalMemoryManager::pageReferenceCountForTest(owned);
311 "shrink fixture did not establish borrowed and private ownership");
312 const bool rejected = !file.resize(pageSize);
313 physical_uintptr_t afterBorrowed = 0;
314 size_t afterBorrowedFlags = 0;
315 physical_uintptr_t afterPrivate = 0;
316 size_t afterPrivateFlags = 0;
317 const bool borrowedPresent = space.
isMapped(
reinterpret_cast<void*
>(shared + pageSize));
318 const bool privatePresent = space.
isMapped(
const_cast<uint8_t*
>(privateByte));
320 space.
getMapping(
reinterpret_cast<void*
>(shared + pageSize), afterBorrowed, afterBorrowedFlags);
322 space.
getMapping(
const_cast<uint8_t*
>(privateByte), afterPrivate, afterPrivateFlags);
324 rejected && !file.shrinkCommits && file.preparedLoans == originalLoans &&
325 file.loans() == originalLoans && file.getSize() == 2 * pageSize && borrowedPresent &&
326 afterBorrowed == borrowed && afterBorrowedFlags == borrowedFlags && privatePresent &&
327 afterPrivate == owned && afterPrivateFlags == privateFlags && *privateByte == 0xA5 &&
328 PhysicalMemoryManager::pageReferenceCountForTest(owned) == privateReferences,
329 "failed backend preparation changed a suffix PTE, loan, or private page");
330 file.rejectResize =
false;
331 const bool resized = file.resize(pageSize);
332 physical_uintptr_t afterPrefix = 0;
333 size_t afterPrefixFlags = 0;
334 const bool prefixPresent = space.
isMapped(
reinterpret_cast<void*
>(shared));
336 space.
getMapping(
reinterpret_cast<void*
>(shared), afterPrefix, afterPrefixFlags);
338 check(resized && file.shrinkCommits == 1 && file.preparedLoans == originalLoans &&
339 file.committedLoans == 1 && file.loans() == 1 && prefixPresent &&
340 afterPrefix == prefix && afterPrefixFlags == prefixFlags &&
341 file.getSize() == pageSize && !space.
isMapped(
const_cast<uint8_t*
>(privateByte)) &&
342 !space.
isMapped(
reinterpret_cast<void*
>(shared + pageSize)) &&
343 !manager.faultIn(privateAddress + pageSize,
false) &&
344 !manager.faultIn(shared + pageSize,
false) &&
345 PhysicalMemoryManager::pageReferenceCountForTest(owned) == 0,
346 "shrink commit lost the prefix or retained suffix ownership");
353bool failedMappedResize() {
356 Thread*
worker =
new Thread(process, resizeFailureWorker, &passed,
nullptr,
false,
true,
true);
357 const bool started =
worker->start();
358 const bool joined = started &&
worker->joinForCompletion();
363 return check(started && joined && passed,
"mapped resize backend failure fixture");
367int sparseSplitWorker(
void* parameter) {
368 bool& passed = *
static_cast<bool*
>(parameter);
373 ResizeProbeFile file(3);
374 if (!file.initialise()) {
378 const unsigned masks[] = {4, 2, 5, 7};
379 for (
unsigned mask : masks) {
380 NOTICE(
"VM-OWNERSHIP-TEST: BEGIN sparse-split mask=" << mask);
381 uintptr_t address = 0;
383 manager.
mapFile(&file, address, 3 * pageSize, MemoryMappedObject::Read, 0,
false);
384 if (!check(
object !=
nullptr,
"sparse file setup")) {
388 size_t expectedLoans = 0;
389 for (
size_t page = 3; page; --page) {
390 if (mask & (1U << (page - 1))) {
391 passed &= check(manager.faultIn(address + (page - 1) * pageSize,
false),
392 "sparse resident page setup");
396 passed &= check(file.loans() == expectedLoans,
"sparse backing loan count before split");
398 manager.setPermissions(address + pageSize, 2 * pageSize, MemoryMappedObject::Read) != 0,
399 "sparse protection split");
400 passed &= check(file.loans() == expectedLoans,
"sparse split changed backing loans");
402 "sparse removal did not visit both objects");
403 passed &= check(!manager.
contains(address, 3 * pageSize),
"sparse objects survived removal");
404 passed &= check(file.loans() == 0,
"sparse removal retained backing loans");
405 for (
size_t page = 0; page < 3; ++page) {
406 passed &= check(!space.
isMapped(
reinterpret_cast<void*
>(address + page * pageSize)),
407 "sparse removal retained a PTE");
409 const uintptr_t requested = address;
410 MemoryMapManager::MapStatus status;
411 object = manager.
mapAnon(address, 3 * pageSize, MemoryMappedObject::Read,
412 MemoryMapManager::Placement::FixedNoReplace, &status);
414 check(
object && address == requested && status == MemoryMapManager::MapStatus::Success,
415 "sparse removal retained a reservation");
417 for (
size_t page = 0; page < 3; ++page) {
418 const uintptr_t at = address + page * pageSize;
419 if (manager.faultIn(at,
false)) {
420 const volatile uint8_t* bytes =
reinterpret_cast<const volatile uint8_t*
>(at);
421 for (
size_t byte = 0;
byte < pageSize; ++byte) {
422 if (!check(bytes[
byte] == 0,
"sparse anonymous reuse retained file contents")) {
423 ERROR(
"VM-OWNERSHIP-TEST: byte=" << page * pageSize +
byte
424 <<
" value=" <<
static_cast<unsigned>(bytes[
byte]));
430 passed &= check(
false,
"sparse anonymous reuse could not fault");
438 NOTICE(
"VM-OWNERSHIP-TEST: PASS sparse-split mask="
439 << mask <<
" objects=0 ptes=0 loans=0 reservation=reused zero=verified");
445bool sparseSplitOwnership() {
448 Thread*
worker =
new Thread(process, sparseSplitWorker, &passed,
nullptr,
false,
true,
true);
449 const bool started =
worker->start();
450 const bool joined = started &&
worker->joinForCompletion();
455 return check(started && joined && passed,
"sparse split ownership fixture");
458int checkedSyncWorker(
void* parameter) {
459 bool& passed = *
static_cast<bool*
>(parameter);
463 ResizeProbeFile file(3);
464 if (!file.initialise()) {
467 uintptr_t address = 0;
469 manager.
mapFile(&file, address, 3 * pageSize, MemoryMappedObject::Read, 0,
false);
470 if (!
object || !manager.faultIn(address,
false) || !manager.faultIn(address + pageSize,
false)) {
476 void* mapping =
reinterpret_cast<void*
>(address);
477 file.rejectSync =
true;
479 passed = check(posix_msync(mapping, 3 * pageSize, MS_SYNC) == -1 &&
480 thread->
getErrno() == Error::OutOfMemory && file.syncCalls == 0,
481 "msync range failure reached backing I/O or lost ENOMEM");
483 passed &= check(posix_msync(mapping, 2 * pageSize, MS_SYNC) == -1 &&
484 thread->
getErrno() == Error::IoError && file.syncCalls == 2,
485 "msync lost EIO or skipped a page after the first backend failure");
486 file.rejectSync =
false;
488 passed &= check(posix_msync(mapping, 2 * pageSize, MS_SYNC) == 0 && file.syncCalls == 4,
489 "msync could not retry both failed pages");
494 "VM-OWNERSHIP-TEST: PASS checked-msync range=ENOMEM range-io=0 failure=EIO "
495 "attempted=2 retry=2");
500bool checkedMappedSync() {
503 Thread*
worker =
new Thread(process, checkedSyncWorker, &passed,
nullptr,
false,
true,
true);
504 const bool started =
worker->start();
505 const bool joined = started &&
worker->joinForCompletion();
510 return check(started && joined && passed,
"checked mapped sync fixture");
514bool runVmMappedOwnershipRegressions() {
515 NOTICE(
"VM-OWNERSHIP-TEST: BEGIN sparse-split-ownership");
516 if (!sparseSplitOwnership()) {
519 NOTICE(
"VM-OWNERSHIP-TEST: PASS sparse-split-ownership cases=4");
520 NOTICE(
"VM-OWNERSHIP-TEST: BEGIN checked-msync");
521 if (!checkedMappedSync()) {
524 NOTICE(
"VM-OWNERSHIP-TEST: PASS mapped-ownership");
528#if HOSTED && PEDIGREE_HOSTED_SMOKE_TESTS
529bool runHostedVmOperationGuardRegressions() {
534 auto protectedState = [&]() {
537 auto restoredState = [&]() {
544 passed &= check(operation && protectedState(),
"mapping operation lost deferral protection");
547 passed &= check(nestedOperation && protectedState(),
548 "nested mapping operation lost deferral protection");
551 passed &= check(!pressure && protectedState(),
552 "failed pressure entry changed mapping operation protection");
554 passed &= check(protectedState(),
"failed pressure guard retired outer protection");
556 passed &= check(protectedState(),
"nested mapping guard retired outer protection");
558 passed &= check(restoredState(),
"mapping operation did not restore prior deferral state");
561 passed &= check(pressure && protectedState(),
562 "successful pressure entry lacked the operation gate or protection");
564 passed &= check(restoredState(),
"pressure guard did not restore prior deferral state");
569 passed &= check(operation && protectedState(),
570 "mapping operation lost enclosing uninterruptible protection");
572 passed &= check(protectedState(),
"mapping guard retired enclosing uninterruptible scope");
574 passed &= check(restoredState(),
"enclosing scope did not restore prior deferral state");
576 NOTICE(
"HOSTED-WAIT-TEST: PASS vm-operation-guard-deferrals");
581bool runHostedVmPermissionRegressions() {
583 bool passed = check(space.
isAddressValid(
reinterpret_cast<void*
>(0x00007FFFFFFFFFFFULL)) &&
584 !space.
isAddressValid(
reinterpret_cast<void*
>(0x0000800000000000ULL)) &&
585 !space.
isAddressValid(
reinterpret_cast<void*
>(0xFFFF7FFFFFFFFFFFULL)) &&
586 space.
isAddressValid(
reinterpret_cast<void*
>(0xFFFF800000000000ULL)),
587 "four-level canonical address boundaries");
588 passed &= runHostedVmOperationGuardRegressions();
589 passed &= protectedClone(
false);
590 passed &= protectedClone(
true);
591 passed &= borrowedClones();
592 passed &= failedMappedResize();
593 passed &= runVmMappedOwnershipRegressions();
595 NOTICE(
"HOSTED-WAIT-TEST: PASS vm-permission-ownership");
Memory-mapped file interface.
virtual uintptr_t readBlock(uint64_t location)
virtual void unpinBlock(uint64_t location)
virtual bool prepareSharedMapping(size_t offset, size_t length)
virtual MUST_USE_RESULT bool pinBlock(uint64_t location)
MemoryMappedObject * mapAnon(uintptr_t &address, size_t length, MemoryMappedObject::Permissions perms)
static MemoryMapManager & instance()
MemoryMappedObject * mapFile(File *pFile, uintptr_t &address, size_t length, MemoryMappedObject::Permissions perms, size_t offset=0, bool bCopyOnWrite=true)
size_t removeAndRelease(uintptr_t base, size_t length, VmStatus *status=nullptr)
bool contains(uintptr_t base, size_t length)
Special memory entity in the kernel's virtual address space.
void * virtualAddress() const
virtual physical_uintptr_t allocatePage(size_t pageConstraints=0)=0
static PhysicalMemoryManager & instance()
static constexpr size_t getPageSize() PURE
virtual void freePage(physical_uintptr_t page)=0
VirtualAddressSpace * getAddressSpace()
static ProcessorInformation & information()
static void switchAddressSpace(VirtualAddressSpace &AddressSpace)
static Scheduler & instance()
void setErrno(size_t err)
bool eventsDeferred() const
bool isTerminationDeferred() const
static UniquePointer< T > adopt(T *pointer)
virtual void setFlags(void *virtualAddress, size_t newFlags)=0
static const size_t CopyOnWrite
static const size_t Borrowed
virtual uintptr_t getDynamicStart() const
virtual bool map(physical_uintptr_t physicalAddress, void *virtualAddress, size_t flags)=0
static const size_t Shared
virtual bool isMapped(void *virtualAddress)=0
virtual bool handleCopyOnWriteFault(void *virtualAddress, bool userMode)=0
static const size_t KernelMode
virtual bool getMapping(void *virtualAddress, physical_uintptr_t &physicalAddress, size_t &flags)=0
static const size_t NoAccess
static const size_t Write
virtual bool detachMapping(void *virtualAddress, physical_uintptr_t &physical, size_t &flags, size_t requiredFlags=0)
static const size_t WriteProtected
virtual bool isAddressValid(void *virtualAddress)=0