#include <VirtualAddressSpace.h>

Inheritance diagram for X64VirtualAddressSpace:

Collaboration diagram for X64VirtualAddressSpace:

Public Member Functions
virtual bool	isAddressValid (void *virtualAddress)

virtual bool	isMapped (void *virtualAddress)

virtual size_t	runtimeMappingPages (uintptr_t base, size_t length)

virtual bool	map (physical_uintptr_t physAddress, void *virtualAddress, size_t flags)

bool	tryMapUserPage (physical_uintptr_t physical, void address, size_t flags, size_t committedTablePages=nullptr) override

bool	tryDetachUserPage (void *address, physical_uintptr_t expected) override

virtual bool	mapHuge (physical_uintptr_t physAddress, void *virtualAddress, size_t count, size_t flags)

virtual bool	getMapping (void *virtualAddress, physical_uintptr_t &physAddress, size_t &flags)

virtual bool	handleCopyOnWriteFault (void *virtualAddress, bool userMode)

ResidentCopyStatus	copyResidentUserPage (uintptr_t userAddress, void *kernelBuffer, size_t bytes, bool write) override

virtual bool	tryWriteUser32 (uintptr_t address, uint32_t value)

virtual bool	tryReadUser32 (uintptr_t address, uint32_t &value)

virtual bool	tryReadUserPointer (uintptr_t address, uintptr_t &value)

virtual bool	tryCompareExchangeUser32 (uintptr_t address, uint32_t &expected, uint32_t desired, bool &exchanged)

virtual void	setFlags (void *virtualAddress, size_t newFlags)

MUST_USE_RESULT bool	trySetFlags (void *virtualAddress, size_t newFlags) override

virtual void	unmap (void *virtualAddress)

virtual bool	detachMapping (void *virtualAddress, physical_uintptr_t &physical, size_t &flags, size_t requiredFlags=0)

RemapStatus	prepareRemap (const PageRemapRequest &request, UniquePointer< PreparedPageRemap > &plan) override

void	setRemapPreparationFailureForTest (ssize_t allocations) override

virtual Stack *	allocateStack ()

virtual Stack *	allocateStack (size_t stackSz)

virtual void	freeStack (Stack *pStack)

virtual bool	memIsInKernelHeap (void *pMem)

virtual bool	memIsInHeap (void *pMem)

virtual void *	getEndOfHeap ()

virtual VirtualAddressSpace *	clone (bool copyOnWrite=true)

virtual void	revertToKernelAddressSpace ()

bool	mapPageStructures (physical_uintptr_t physAddress, void *virtualAddress, size_t flags)

bool	mapPageStructuresAbove4GB (physical_uintptr_t physAddress, void *virtualAddress, size_t flags)

virtual	~X64VirtualAddressSpace ()

virtual uintptr_t	getKernelStart () const

virtual uintptr_t	getUserStart () const

virtual uintptr_t	getUserReservedStart () const

virtual uintptr_t	getDynamicLinkerAddress () const

virtual uintptr_t	getKernelHeapStart () const

virtual uintptr_t	getKernelHeapEnd () const

virtual uintptr_t	getDynamicStart () const

virtual uintptr_t	getDynamicEnd () const

virtual uintptr_t	getGlobalInfoBlock () const

virtual uintptr_t	getKernelCacheStart () const

virtual uintptr_t	getKernelCacheEnd () const

virtual uintptr_t	getKernelEventBlockStart () const

virtual uintptr_t	getKernelModulesStart () const

virtual uintptr_t	getKernelModulesEnd () const

virtual uintptr_t	getKernelVirtualPagestack () const

virtual uintptr_t	getKernelVirtualPagestackAdd1 () const

virtual uintptr_t	getKernelVirtualPagestackAdd2 () const

Public Member Functions inherited from VirtualAddressSpace
RawUserMemory &	rawUserMemory ()

MemoryLockAccount *	memoryLockAccount () const

void	setMemoryLockAccount (MemoryLockAccount *account)

UserMemoryPolicy *	userMemoryPolicy () const

void	setUserMemoryPolicy (UserMemoryPolicy *policy)

virtual void *	expandHeap (ssize_t incr, size_t flags)

virtual	~VirtualAddressSpace ()

void	setHeap (void heap, void heapEnd)

Private Member Functions
bool	tryAccessUserWord (uintptr_t address, size_t width, uintptr_t &value, const uintptr_t *replacement)

	X64VirtualAddressSpace ()

	X64VirtualAddressSpace (void Heap, physical_uintptr_t PhysicalPML4, void VirtualStack)

	X64VirtualAddressSpace (const X64VirtualAddressSpace &)

X64VirtualAddressSpace &	operator= (const X64VirtualAddressSpace &)

bool	getPageTableEntry (void virtualAddress, uint64_t &pageTableEntry) const

size_t	detachEmptyTables (void virtualAddress, physical_uintptr_t detachedTables)

bool	invalidateMapping (void *virtualAddress, X64MappingMutationScope &mutation)

uint64_t	toFlags (size_t flags, void *virtualAddress, bool bFinal=false) const PURE

size_t	fromFlags (uint64_t Flags, bool bFinal=false) const PURE

bool	conditionalTableEntryAllocation (uint64_t *tableEntry, uint64_t flags)

bool	conditionalTableEntryMapping (uint64_t *tableEntry, uint64_t physAddress, uint64_t flags)

bool	mapUnlocked (physical_uintptr_t physAddress, void *virtualAddress, size_t flags, X64MappingMutationScope &mutation, bool locked=false)

bool	unmapUnlocked (void *virtualAddress, X64MappingMutationScope &mutation, bool requireMapped=true)

Stack *	doAllocateStack (size_t sSize)

Stack *	allocateTrackedUserStack (size_t size)

Private Attributes
physical_uintptr_t	m_PhysicalPML4

Atomic< uint64_t >	m_ResidentProcessors

ssize_t	m_RemapPreparationFailure = -1

void *	m_pStackTop

Vector< Stack * >	m_freeStacks

bool	m_bKernelSpace

Spinlock	m_Lock

Spinlock	m_StacksLock

Static Private Attributes
static X64VirtualAddressSpace	m_KernelSpace

Friends
class	ProcessorBase

class	Multiprocessor

class	X64PreparedPageRemap

VirtualAddressSpace &	VirtualAddressSpace::getKernelAddressSpace ()

VirtualAddressSpace *	VirtualAddressSpace::create ()

Additional Inherited Members
Public Types inherited from VirtualAddressSpace
enum class	ResidentCopyStatus { Success , Inaccessible , Unsupported }

enum class	RemapStatus { Success , InvalidRange , Unsupported , NoMemory , Retry }

using	RemapAdmission = bool()(void context)

Static Public Member Functions inherited from VirtualAddressSpace
static EXPORTED_PUBLIC VirtualAddressSpace &	getKernelAddressSpace ()

static VirtualAddressSpace *	create ()

Public Attributes inherited from VirtualAddressSpace
void *	m_Heap

void *	m_HeapEnd

Static Public Attributes inherited from VirtualAddressSpace
static const size_t	KernelMode = 0x01

static const size_t	Write = 0x02

static const size_t	Execute = 0x04

static const size_t	WriteThrough = 0x08

static const size_t	CacheDisable = 0x10

static const size_t	CopyOnWrite = 0x20

static const size_t	Swapped = 0x40

static const size_t	Shared = 0x200

static const size_t	WriteCombine = 0x400

static const size_t	Accessed = 0x800

static const size_t	Dirty = 0x1000

static const size_t	ClearDirty = 0x2000

static const size_t	NoAccess = 0x4000

static const size_t	WriteProtected = 0x8000

static const size_t	Borrowed = 0x10000

static const size_t	RuntimeMapping = 0x20000

static constexpr size_t	MaximumRemapPages = 65536

static physical_uintptr_t	m_ZeroPage = 0

Protected Member Functions inherited from VirtualAddressSpace
bool	admitRawMemoryChange (const PreparedMemoryLock &plan, bool privileged, MemoryLockCharge &charge) const

void	commitRawMemoryChange (PreparedMemoryLock &plan, MemoryLockCharge charge)

bool	prepareZeroPage ()

	VirtualAddressSpace (void *Heap)

Protected Attributes inherited from VirtualAddressSpace
uint64_t	m_HeapRegionId = 0

Detailed Description

The X64VirtualAddressSpace implements the VirtualAddressSpace class for the x64 processor architecture, that means it wraps around the processor's paging functionality.

Definition at line 87 of file kernel/core/processor/x64/VirtualAddressSpace.h.

Constructor & Destructor Documentation

◆ ~X64VirtualAddressSpace()

X64VirtualAddressSpace::~X64VirtualAddressSpace ( )

virtual

The destructor cleans up the address space

Todo:: validate that we're cleaning up enough stuff here for example are we missing stuff like PDPTs etc?

Definition at line 1229 of file x64/VirtualAddressSpace.cc.

References PhysicalMemoryManager::freePage(), PhysicalMemoryManager::instance(), m_bKernelSpace, m_PhysicalPML4, m_ResidentProcessors, and revertToKernelAddressSpace().

◆ X64VirtualAddressSpace() [1/3]

X64VirtualAddressSpace::X64VirtualAddressSpace ( )

private

The default constructor

Definition at line 1244 of file x64/VirtualAddressSpace.cc.

References PhysicalMemoryManager::allocatePage(), PhysicalMemoryManager::instance(), m_KernelSpace, m_PhysicalPML4, and physicalAddress().

◆ X64VirtualAddressSpace() [2/3]

X64VirtualAddressSpace::X64VirtualAddressSpace	(	void *	Heap,
		physical_uintptr_t	PhysicalPML4,
		void *	VirtualStack
	)

private

The constructor for already present paging structures

Parameters

[in]	Heap	virtual address of the beginning of the heap
[in]	PhysicalPageDirectory	physical address of the page directory
[in]	VirtualStack	virtual address of the top of the next kernel stack

Note: This constructor is only used to construct the kernel VirtualAddressSpace

Definition at line 1265 of file x64/VirtualAddressSpace.cc.

◆ X64VirtualAddressSpace() [3/3]

X64VirtualAddressSpace::X64VirtualAddressSpace ( const X64VirtualAddressSpace & )

private

The copy-constructor

Note: NOT implemented

Member Function Documentation

◆ allocateStack() [1/2]

VirtualAddressSpace::Stack * X64VirtualAddressSpace::allocateStack ( )

virtual

Allocates a single stack for a thread. Will use the default kernel thread size.

Implements VirtualAddressSpace.

Definition at line 1020 of file x64/VirtualAddressSpace.cc.

References doAllocateStack(), and m_KernelSpace.

Referenced by allocateStack().

Here is the caller graph for this function:

◆ allocateStack() [2/2]

VirtualAddressSpace::Stack * X64VirtualAddressSpace::allocateStack ( size_t stackSz )

virtual

Allocates a single stack of the given size for a thread.

Reimplemented from VirtualAddressSpace.

Definition at line 1027 of file x64/VirtualAddressSpace.cc.

References allocateStack(), and doAllocateStack().

◆ allocateTrackedUserStack()

VirtualAddressSpace::Stack * X64VirtualAddressSpace::allocateTrackedUserStack ( size_t size )

private

Definition at line 1105 of file x64/VirtualAddressSpace.cc.

◆ clone()

VirtualAddressSpace * X64VirtualAddressSpace::clone ( bool copyOnWrite = true )

virtual

Clone this VirtualAddressSpace. That means that we copy-on-write-map the application * image. *

Parameters

[in] copyOnWrite whether the new address space copies pages on write, or shares them with the parent address space.

Returns: pointer to the new VirtualAddressSpace

Todo:: figure out how to handle page tracking here

Todo:: Deal with 2MB pages here.

Implements VirtualAddressSpace.

Definition at line 650 of file x64/VirtualAddressSpace.cc.

References Spinlock::acquire(), Vector< T >::begin(), VirtualAddressSpace::create(), Vector< T >::end(), fromFlags(), PhysicalMemoryManager::instance(), invalidateMapping(), m_freeStacks, VirtualAddressSpace::m_Heap, VirtualAddressSpace::m_HeapEnd, m_Lock, m_PhysicalPML4, m_pStackTop, m_StacksLock, mapUnlocked(), physicalAddress(), PhysicalMemoryManager::pin(), Vector< T >::pushBack(), and Spinlock::release().

◆ conditionalTableEntryAllocation()

bool X64VirtualAddressSpace::conditionalTableEntryAllocation	(	uint64_t *	tableEntry,
		uint64_t	flags
	)

private

Allocate and map the table entry if none is present

Parameters

[in]	tableEntry	pointer to the current table entry
[in]	flags	flags that are used for the mapping

Returns: true, if successfull, false otherwise

Definition at line 1542 of file x64/VirtualAddressSpace.cc.

References PhysicalMemoryManager::allocatePage(), PhysicalMemoryManager::getPageSize(), PhysicalMemoryManager::instance(), physicalAddress(), and toFlags().

Referenced by mapHuge(), and mapUnlocked().

Here is the caller graph for this function:

◆ conditionalTableEntryMapping()

bool X64VirtualAddressSpace::conditionalTableEntryMapping	(	uint64_t *	tableEntry,
		uint64_t	physAddress,
		uint64_t	flags
	)

private

map the table entry if none is present

Parameters

[in]	tableEntry	pointer to the current table entry
[in]	physAddress	physical address of the page used as the table
[in]	flags	flags that are used for the mapping

Returns: true, if the page has been used, false otherwise

Definition at line 1577 of file x64/VirtualAddressSpace.cc.

References PhysicalMemoryManager::getPageSize(), physicalAddress(), and toFlags().

Referenced by mapPageStructures().

Here is the caller graph for this function:

◆ copyResidentUserPage()

VirtualAddressSpace::ResidentCopyStatus X64VirtualAddressSpace::copyResidentUserPage	(	uintptr_t	userAddress,
		void *	kernelBuffer,
		size_t	bytes,
		bool	write
	)

overridevirtual

One-page, no-fault copy with a resident nonpageable kernel buffer.

Reimplemented from VirtualAddressSpace.

Definition at line 434 of file x64/VirtualAddressSpace.cc.

References getKernelStart(), PhysicalMemoryManager::getPageSize(), getUserStart(), m_Lock, m_PhysicalPML4, and physicalAddress().

◆ detachEmptyTables()

size_t X64VirtualAddressSpace::detachEmptyTables	(	void *	virtualAddress,
		physical_uintptr_t *	detachedTables
	)

private

Detach empty paging structures for the given address without freeing their physical storage.

The caller invalidates the address before releasing the returned pages, so no processor can retain a paging-structure-cache reference into reused storage.

Definition at line 1337 of file x64/VirtualAddressSpace.cc.

References Process::getAddressSpace(), Thread::getParent(), ProcessorBase::information(), and m_PhysicalPML4.

◆ detachMapping()

bool X64VirtualAddressSpace::detachMapping	(	void *	virtualAddress,
		physical_uintptr_t &	physical,
		size_t &	flags,
		size_t	requiredFlags = `0`
	)

virtual

Atomically snapshot and detach a resident mapping. A flags mismatch leaves the mapping intact, but still returns its physical address and flags.

Reimplemented from VirtualAddressSpace.

Definition at line 602 of file x64/VirtualAddressSpace.cc.

References fromFlags(), getPageTableEntry(), m_Lock, and unmapUnlocked().

◆ doAllocateStack()

VirtualAddressSpace::Stack * X64VirtualAddressSpace::doAllocateStack ( size_t sSize )

private

Allocates a stack with a given size.

Definition at line 1033 of file x64/VirtualAddressSpace.cc.

References Spinlock::acquire(), PhysicalMemoryManager::allocatePage(), VirtualAddressSpace::CopyOnWrite, Vector< T >::count(), ProcessorBase::getInterrupts(), PhysicalMemoryManager::getPageSize(), PhysicalMemoryManager::instance(), VirtualAddressSpace::KernelMode, m_freeStacks, m_KernelSpace, m_Lock, m_pStackTop, m_StacksLock, map(), PhysicalMemoryManager::pin(), Vector< T >::popBack(), Spinlock::release(), and VirtualAddressSpace::Write.

Referenced by allocateStack(), and allocateStack().

Here is the caller graph for this function:

◆ freeStack()

void X64VirtualAddressSpace::freeStack ( Stack * pStack )

virtual

Frees a stack allocated with allocateStack.

Implements VirtualAddressSpace.

Definition at line 1187 of file x64/VirtualAddressSpace.cc.

References Spinlock::acquire(), PhysicalMemoryManager::freePage(), getMapping(), PhysicalMemoryManager::getPageSize(), PhysicalMemoryManager::instance(), isMapped(), m_freeStacks, m_StacksLock, Vector< T >::pushBack(), Spinlock::release(), and unmap().

◆ fromFlags()

size_t X64VirtualAddressSpace::fromFlags	(	uint64_t	Flags,
		bool	bFinal = `false`
	)		const

private

Convert processor's representation of the flags to the processor independant representation

Parameters

[in]	Flags	the processor specific flag representation
[in]	bFinal	whether this is for the actual page or just an intermediate PTE/PDE

Returns: the proessor independant flag representation

Definition at line 1505 of file x64/VirtualAddressSpace.cc.

References VirtualAddressSpace::Accessed, VirtualAddressSpace::Borrowed, VirtualAddressSpace::CacheDisable, VirtualAddressSpace::CopyOnWrite, VirtualAddressSpace::Dirty, VirtualAddressSpace::Execute, VirtualAddressSpace::KernelMode, VirtualAddressSpace::NoAccess, VirtualAddressSpace::RuntimeMapping, VirtualAddressSpace::Shared, VirtualAddressSpace::Swapped, VirtualAddressSpace::Write, VirtualAddressSpace::WriteCombine, VirtualAddressSpace::WriteProtected, and VirtualAddressSpace::WriteThrough.

Referenced by clone(), detachMapping(), and getMapping().

Here is the caller graph for this function:

◆ getDynamicEnd()

virtual uintptr_t X64VirtualAddressSpace::getDynamicEnd ( ) const

inlinevirtual

Gets address of the end of dynamic memory mapping area.

Reimplemented from VirtualAddressSpace.

Definition at line 198 of file kernel/core/processor/x64/VirtualAddressSpace.h.

Referenced by prepareRemap().

Here is the caller graph for this function:

◆ getDynamicLinkerAddress()

virtual uintptr_t X64VirtualAddressSpace::getDynamicLinkerAddress ( ) const

inlinevirtual

Gets address of the dynamic linker in the address space.

Implements VirtualAddressSpace.

Definition at line 178 of file kernel/core/processor/x64/VirtualAddressSpace.h.

◆ getDynamicStart()

virtual uintptr_t X64VirtualAddressSpace::getDynamicStart ( ) const

inlinevirtual

Gets address of the start of dynamic memory mapping area.

Reimplemented from VirtualAddressSpace.

Definition at line 193 of file kernel/core/processor/x64/VirtualAddressSpace.h.

Referenced by prepareRemap().

Here is the caller graph for this function:

◆ getEndOfHeap()

void * X64VirtualAddressSpace::getEndOfHeap ( )

virtual

Gets a pointer to the byte after the end of the heap.

Implements VirtualAddressSpace.

Definition at line 85 of file x64/VirtualAddressSpace.cc.

References VirtualAddressSpace::m_Heap, and VirtualAddressSpace::m_HeapEnd.

Referenced by memIsInHeap().

Here is the caller graph for this function:

◆ getGlobalInfoBlock()

virtual uintptr_t X64VirtualAddressSpace::getGlobalInfoBlock ( ) const

inlinevirtual

Gets address of the global info block location.

Reimplemented from VirtualAddressSpace.

Definition at line 203 of file kernel/core/processor/x64/VirtualAddressSpace.h.

◆ getKernelCacheEnd()

virtual uintptr_t X64VirtualAddressSpace::getKernelCacheEnd ( ) const

inlinevirtual

Gets address of the end of the kernel's cache region.

Implements VirtualAddressSpace.

Definition at line 213 of file kernel/core/processor/x64/VirtualAddressSpace.h.

◆ getKernelCacheStart()

virtual uintptr_t X64VirtualAddressSpace::getKernelCacheStart ( ) const

inlinevirtual

Gets address of the start of the kernel's cache region.

Implements VirtualAddressSpace.

Definition at line 208 of file kernel/core/processor/x64/VirtualAddressSpace.h.

◆ getKernelEventBlockStart()

virtual uintptr_t X64VirtualAddressSpace::getKernelEventBlockStart ( ) const

inlinevirtual

Gets address of the start of the kernel's event handling block.

Implements VirtualAddressSpace.

Definition at line 218 of file kernel/core/processor/x64/VirtualAddressSpace.h.

◆ getKernelHeapEnd()

virtual uintptr_t X64VirtualAddressSpace::getKernelHeapEnd ( ) const

inlinevirtual

Gets address of the end of the kernel's heap region.

Implements VirtualAddressSpace.

Definition at line 188 of file kernel/core/processor/x64/VirtualAddressSpace.h.

◆ getKernelHeapStart()

virtual uintptr_t X64VirtualAddressSpace::getKernelHeapStart ( ) const

inlinevirtual

Gets address of the start of the kernel's heap region.

Implements VirtualAddressSpace.

Definition at line 183 of file kernel/core/processor/x64/VirtualAddressSpace.h.

◆ getKernelModulesEnd()

virtual uintptr_t X64VirtualAddressSpace::getKernelModulesEnd ( ) const

inlinevirtual

Gets address of the end of the kernel's module region.

Implements VirtualAddressSpace.

Definition at line 228 of file kernel/core/processor/x64/VirtualAddressSpace.h.

◆ getKernelModulesStart()

virtual uintptr_t X64VirtualAddressSpace::getKernelModulesStart ( ) const

inlinevirtual

Gets address of the start of the kernel's module region.

Implements VirtualAddressSpace.

Definition at line 223 of file kernel/core/processor/x64/VirtualAddressSpace.h.

◆ getKernelStart()

virtual uintptr_t X64VirtualAddressSpace::getKernelStart ( ) const

inlinevirtual

Gets start address of the kernel in the address space.

Implements VirtualAddressSpace.

Definition at line 163 of file kernel/core/processor/x64/VirtualAddressSpace.h.

Referenced by copyResidentUserPage(), and tryWriteUser32().

Here is the caller graph for this function:

◆ getKernelVirtualPagestack()

virtual uintptr_t X64VirtualAddressSpace::getKernelVirtualPagestack ( ) const

inlinevirtual

Gets address of the physical page stack, if one exists.

Reimplemented from VirtualAddressSpace.

Definition at line 233 of file kernel/core/processor/x64/VirtualAddressSpace.h.

◆ getKernelVirtualPagestackAdd1()

virtual uintptr_t X64VirtualAddressSpace::getKernelVirtualPagestackAdd1 ( ) const

inlinevirtual

Gets address of the first additional page stack, if one exists.

Reimplemented from VirtualAddressSpace.

Definition at line 238 of file kernel/core/processor/x64/VirtualAddressSpace.h.

◆ getKernelVirtualPagestackAdd2()

virtual uintptr_t X64VirtualAddressSpace::getKernelVirtualPagestackAdd2 ( ) const

inlinevirtual

Gets address of the second additional page stack, if one exists.

Reimplemented from VirtualAddressSpace.

Definition at line 243 of file kernel/core/processor/x64/VirtualAddressSpace.h.

◆ getMapping()

bool X64VirtualAddressSpace::getMapping	(	void *	virtualAddress,
		physical_uintptr_t &	physicalAddress,
		size_t &	flags
	)

virtual

Query the physical address and flags of a mapped or swapped-out page. Returns false without modifying the outputs when no mapping exists. The result does not pin the page; callers must protect its lifetime while using the returned physical address.

Implements VirtualAddressSpace.

Definition at line 314 of file x64/VirtualAddressSpace.cc.

References fromFlags(), and getPageTableEntry().

Referenced by freeStack().

Here is the caller graph for this function:

◆ getPageTableEntry()

bool X64VirtualAddressSpace::getPageTableEntry	(	void *	virtualAddress,
		uint64_t *&	pageTableEntry
	)		const

private

Get the page table entry, if it exists and check whether a page is mapped or marked as swapped out.

Parameters

[in]	virtualAddress	the virtual address
[out]	pageTableEntry	pointer to the page table entry

Returns: true, if the page table is present and the page mapped or marked swapped out, false otherwise

Definition at line 1276 of file x64/VirtualAddressSpace.cc.

References m_PhysicalPML4.

Referenced by detachMapping(), getMapping(), handleCopyOnWriteFault(), setFlags(), trySetFlags(), tryWriteUser32(), and unmapUnlocked().

Here is the caller graph for this function:

◆ getUserReservedStart()

virtual uintptr_t X64VirtualAddressSpace::getUserReservedStart ( ) const

inlinevirtual

Gets start address of reserved areas of the userpace address space.

Implements VirtualAddressSpace.

Definition at line 173 of file kernel/core/processor/x64/VirtualAddressSpace.h.

Referenced by prepareRemap().

Here is the caller graph for this function:

◆ getUserStart()

virtual uintptr_t X64VirtualAddressSpace::getUserStart ( ) const

inlinevirtual

Gets start address of the region usable and cloneable for userspace.

Implements VirtualAddressSpace.

Definition at line 168 of file kernel/core/processor/x64/VirtualAddressSpace.h.

Referenced by copyResidentUserPage(), prepareRemap(), and tryWriteUser32().

Here is the caller graph for this function:

◆ handleCopyOnWriteFault()

bool X64VirtualAddressSpace::handleCopyOnWriteFault	(	void *	virtualAddress,
		bool	userMode
	)

virtual

Resolve a write-protection fault on a copy-on-write mapping.

The replacement page is initialised before it becomes visible. Concurrent callers revalidate the mapping and treat an already-writable replacement as a successfully handled stale fault when the faulting privilege level may access it.

Parameters

[in] userMode true when the faulting access came from user mode.

Implements VirtualAddressSpace.

Definition at line 330 of file x64/VirtualAddressSpace.cc.

References PhysicalMemoryManager::allocatePage(), PhysicalMemoryManager::freePage(), PhysicalMemoryManager::getPageSize(), getPageTableEntry(), PhysicalMemoryManager::instance(), invalidateMapping(), m_Lock, page_align(), and physicalAddress().

◆ invalidateMapping()

bool X64VirtualAddressSpace::invalidateMapping	(	void *	virtualAddress,
		X64MappingMutationScope &	mutation
	)

private

Invalidate an address under a lease acquired before its first PTE write. Private address spaces have no processor-residency mask to narrow the destination set safely.

Definition at line 1314 of file x64/VirtualAddressSpace.cc.

References Process::getAddressSpace(), Thread::getParent(), ProcessorBase::information(), m_bKernelSpace, and m_ResidentProcessors.

Referenced by clone(), X64PreparedPageRemap::commit(), handleCopyOnWriteFault(), mapHuge(), mapUnlocked(), revertToKernelAddressSpace(), setFlags(), trySetFlags(), and unmapUnlocked().

Here is the caller graph for this function:

◆ isAddressValid()

bool X64VirtualAddressSpace::isAddressValid ( void * virtualAddress )

virtual

Is a particular virtual address valid?

Parameters

[in] virtualAddress the virtual address to check

Returns: true, if the address is valid, false otherwise

Implements VirtualAddressSpace.

Definition at line 94 of file x64/VirtualAddressSpace.cc.

◆ isMapped()

bool X64VirtualAddressSpace::isMapped ( void * virtualAddress )

virtual

Checks whether a mapping the the specific virtual address exists. Pages marked as swapped out are not considered mapped; retained NoAccess pages are.

Note: This function must be valid on all the valid addresses within the virtual address space.

Parameters

[in] virtualAddress the virtual address

Returns: true, if a mapping exists, false otherwise

Implements VirtualAddressSpace.

Definition at line 101 of file x64/VirtualAddressSpace.cc.

References m_Lock, and m_PhysicalPML4.

Referenced by freeStack().

Here is the caller graph for this function:

◆ map()

bool X64VirtualAddressSpace::map	(	physical_uintptr_t	physicalAddress,
		void *	virtualAddress,
		size_t	flags
	)

virtual

Map a specific physical page (of size PhysicalMemoryManager::getPageSize()) at a specific location into the virtual address space.

Note: This function must also work on pages marked as swapped out.

Parameters

[in]	physicalAddress	the address of the physical page that should be mapped into the virtual address space.
[in]	virtualAddress	the virtual address at which the page apears within the virtual address space.
[in]	flags	flags that describe which accesses should be allowed on the page.

Returns: true, if successfull, false otherwise

Implements VirtualAddressSpace.

Definition at line 139 of file x64/VirtualAddressSpace.cc.

References m_Lock, and mapUnlocked().

Referenced by doAllocateStack().

Here is the caller graph for this function:

◆ mapHuge()

bool X64VirtualAddressSpace::mapHuge	(	physical_uintptr_t	physAddress,
		void *	virtualAddress,
		size_t	count,
		size_t	flags
	)

virtual

Map a region of memory using the largest possible frame size. Where possible the largest page size should be used, degrading into smaller page sizes as needed (e.g. 1 GB, 2 MB, 4K) to reduce the overhead of the mapping on the system. Default implementation calls map() in a loop.

Returns: true if successful, false otherwise.

Reimplemented from VirtualAddressSpace.

Definition at line 151 of file x64/VirtualAddressSpace.cc.

References conditionalTableEntryAllocation(), PhysicalMemoryManager::freePage(), PhysicalMemoryManager::getPageSize(), PhysicalMemoryManager::instance(), invalidateMapping(), m_Lock, m_PhysicalPML4, VirtualAddressSpace::mapHuge(), mapHuge(), toFlags(), and unmapUnlocked().

Referenced by mapHuge().

Here is the caller graph for this function:

◆ mapPageStructures()

bool X64VirtualAddressSpace::mapPageStructures	(	physical_uintptr_t	physAddress,
		void *	virtualAddress,
		size_t	flags
	)

Map the page directory pointer table, the page directory, the page table or the page frame if none is currently present

Note: This should only be used from the PhysicalMemoryManager

Parameters

[in]	physAddress	the physical page that should be used as page table or page frame
[in]	virtualAddress	the virtual address that should be checked for the existance of a page directory pointer table, page directory, page table or page frame
[in]	flags	the flags used for the mapping

Returns: true, if a page table/frame is already mapped for that address, false if the physicalAddress has been used as a page directory pointer table, page directory, page table or page frame.

Definition at line 899 of file x64/VirtualAddressSpace.cc.

References conditionalTableEntryMapping(), ProcessorBase::m_Initialised, m_Lock, m_PhysicalPML4, panic(), and toFlags().

Referenced by X86CommonPhysicalMemoryManager::PageStack::maybeMap().

Here is the caller graph for this function:

◆ mapPageStructuresAbove4GB()

bool X64VirtualAddressSpace::mapPageStructuresAbove4GB	(	physical_uintptr_t	physAddress,
		void *	virtualAddress,
		size_t	flags
	)

Definition at line 945 of file x64/VirtualAddressSpace.cc.

◆ mapUnlocked()

bool X64VirtualAddressSpace::mapUnlocked	(	physical_uintptr_t	physAddress,
		void *	virtualAddress,
		size_t	flags,
		X64MappingMutationScope &	mutation,
		bool	locked = `false`
	)

private

Perform a mapping proper, without taking the lock.

Parameters

[in] locked whether the lock was taken before calling or not.

Definition at line 228 of file x64/VirtualAddressSpace.cc.

References Scheduler::acquireProcess(), conditionalTableEntryAllocation(), Process::getAddressSpace(), Scheduler::getNumProcesses(), Scheduler::instance(), invalidateMapping(), ProcessorBase::m_Initialised, m_Lock, m_PhysicalPML4, and toFlags().

Referenced by clone(), and map().

Here is the caller graph for this function:

◆ memIsInHeap()

bool X64VirtualAddressSpace::memIsInHeap ( void * pMem )

virtual

Determines if a given point is within the virtual address space region dedicated to the kernel heap.

Implements VirtualAddressSpace.

Definition at line 74 of file x64/VirtualAddressSpace.cc.

References getEndOfHeap(), and VirtualAddressSpace::m_Heap.

◆ memIsInKernelHeap()

bool X64VirtualAddressSpace::memIsInKernelHeap ( void * pMem )

virtual

Determines whether a given address is within the kernel's heap region.

Implements VirtualAddressSpace.

Definition at line 64 of file x64/VirtualAddressSpace.cc.

◆ operator=()

X64VirtualAddressSpace & X64VirtualAddressSpace::operator= ( const X64VirtualAddressSpace & )

private

The copy-constructor

Note: Not implemented

◆ prepareRemap()

VirtualAddressSpace::RemapStatus X64VirtualAddressSpace::prepareRemap	(	const PageRemapRequest &	request,
		UniquePointer< PreparedPageRemap > &	plan
	)

overridevirtual

Preparation changes no live leaf or reservation; Retry changes neither.

Reimplemented from VirtualAddressSpace.

Definition at line 270 of file VirtualAddressSpace-remap.cc.

References UniquePointer< T >::adopt(), getDynamicEnd(), getDynamicStart(), getUserReservedStart(), getUserStart(), and m_bKernelSpace.

◆ revertToKernelAddressSpace()

void X64VirtualAddressSpace::revertToKernelAddressSpace ( )

virtual

Undo a clone() - this happens when an application is Exec()'d - we destroy all mappings not in the kernel address space so the space is 'clean'.

Todo:: Deal with 2MB pages here.

Todo:: When swap system comes along, we want to remove this page from swap!

Implements VirtualAddressSpace.

Definition at line 799 of file x64/VirtualAddressSpace.cc.

References PhysicalMemoryManager::freePage(), PhysicalMemoryManager::instance(), invalidateMapping(), VirtualAddressSpace::m_Heap, VirtualAddressSpace::m_HeapEnd, m_Lock, m_PhysicalPML4, and physicalAddress().

Referenced by ~X64VirtualAddressSpace().

Here is the caller graph for this function:

◆ runtimeMappingPages()

size_t X64VirtualAddressSpace::runtimeMappingPages	(	uintptr_t	base,
		size_t	length
	)

virtual

Reimplemented from VirtualAddressSpace.

Definition at line 988 of file x64/VirtualAddressSpace.cc.

◆ setFlags()

void X64VirtualAddressSpace::setFlags	(	void *	virtualAddress,
		size_t	newFlags
	)

virtual

Set the flags of the page at a specific virtual address.

Note: The page must have been mapped with VirtualAddressSpace::map() and the page must still be mapped or marked as swapped out.

Parameters

[in]	virtualAddress	the virtual address
[in]	newFlags	the flags

Implements VirtualAddressSpace.

Definition at line 548 of file x64/VirtualAddressSpace.cc.

References getPageTableEntry(), invalidateMapping(), m_Lock, and toFlags().

◆ setRemapPreparationFailureForTest()

void X64VirtualAddressSpace::setRemapPreparationFailureForTest ( ssize_t allocations )

inlineoverridevirtual

One-shot failure after the given number of successful preparation allocations.

Reimplemented from VirtualAddressSpace.

Definition at line 128 of file kernel/core/processor/x64/VirtualAddressSpace.h.

◆ toFlags()

uint64_t X64VirtualAddressSpace::toFlags	(	size_t	flags,
		void *	virtualAddress,
		bool	bFinal = `false`
	)		const

private

Convert the processor independant flags to the processor's representation of the flags

Parameters

[in]	flags	the processor independant flag representation
[in]	bFinal	whether this is for the actual page or just an intermediate PTE/PDE

Returns: the proessor specific flag representation

Definition at line 1453 of file x64/VirtualAddressSpace.cc.

Referenced by conditionalTableEntryAllocation(), conditionalTableEntryMapping(), mapHuge(), mapPageStructures(), mapUnlocked(), setFlags(), and trySetFlags().

Here is the caller graph for this function:

◆ tryAccessUserWord()

bool X64VirtualAddressSpace::tryAccessUserWord	(	uintptr_t	address,
		size_t	width,
		uintptr_t &	value,
		const uintptr_t *	replacement
	)

private

Definition at line 474 of file x64/VirtualAddressSpace.cc.

◆ tryCompareExchangeUser32()

bool X64VirtualAddressSpace::tryCompareExchangeUser32	(	uintptr_t	address,
		uint32_t &	expected,
		uint32_t	desired,
		bool &	exchanged
	)

virtual

A successful access may still report a failed comparison.

Reimplemented from VirtualAddressSpace.

Definition at line 420 of file x64/VirtualAddressSpace.cc.

◆ tryDetachUserPage()

bool X64VirtualAddressSpace::tryDetachUserPage	(	void *	address,
		physical_uintptr_t	expected
	)

overridevirtual

Reimplemented from VirtualAddressSpace.

Definition at line 83 of file VirtualAddressSpace-paging.cc.

◆ tryMapUserPage()

bool X64VirtualAddressSpace::tryMapUserPage	(	physical_uintptr_t	physical,
		void *	address,
		size_t	flags,
		size_t *	committedTablePages = `nullptr`
	)

overridevirtual

Reimplemented from VirtualAddressSpace.

Definition at line 8 of file VirtualAddressSpace-paging.cc.

◆ tryReadUser32()

bool X64VirtualAddressSpace::tryReadUser32	(	uintptr_t	address,
		uint32_t &	value
	)

virtual

Atomic, no-fault accesses through a resident userspace mapping.

Reimplemented from VirtualAddressSpace.

Definition at line 407 of file x64/VirtualAddressSpace.cc.

◆ tryReadUserPointer()

bool X64VirtualAddressSpace::tryReadUserPointer	(	uintptr_t	address,
		uintptr_t &	value
	)

virtual

Reimplemented from VirtualAddressSpace.

Definition at line 416 of file x64/VirtualAddressSpace.cc.

◆ trySetFlags()

bool X64VirtualAddressSpace::trySetFlags	(	void *	virtualAddress,
		size_t	newFlags
	)

overridevirtual

Changes an existing mapping, returning false without publishing failed protection.

Reimplemented from VirtualAddressSpace.

Definition at line 568 of file x64/VirtualAddressSpace.cc.

References getPageTableEntry(), invalidateMapping(), m_Lock, and toFlags().

◆ tryWriteUser32()

bool X64VirtualAddressSpace::tryWriteUser32	(	uintptr_t	address,
		uint32_t	value
	)

virtual

Stores one 32-bit value through a resident userspace mapping without switching into this address space. Implementations reject mappings that could fault or change backing storage while the write is in progress.

Reimplemented from VirtualAddressSpace.

Definition at line 518 of file x64/VirtualAddressSpace.cc.

References getKernelStart(), PhysicalMemoryManager::getPageSize(), getPageTableEntry(), getUserStart(), m_Lock, and physicalAddress().