Inheritance diagram for Arm64VirtualAddressSpace:

Collaboration diagram for Arm64VirtualAddressSpace:

Public Member Functions
bool	isAddressValid (void *address) override

bool	isMapped (void *address) override

bool	map (physical_uintptr_t physical, void *address, size_t flags) override

bool	tryMapUserPage (physical_uintptr_t physical, void address, size_t flags, size_t committedTablePages=nullptr) override

bool	tryDetachUserPage (void *address, physical_uintptr_t expected) override

bool	getMapping (void *address, physical_uintptr_t &physical, size_t &flags) override

bool	handleCopyOnWriteFault (void *address, bool userMode) override

void	setFlags (void *address, size_t flags) override

bool	trySetFlags (void *address, size_t flags) override

void	unmap (void *address) override

bool	detachMapping (void *address, physical_uintptr_t &physical, size_t &flags, size_t requiredFlags=0) override

Stack *	allocateStack () override

Stack *	allocateStack (size_t bytes) override

void	freeStack (Stack *stack) override

VirtualAddressSpace *	clone (bool copyOnWrite=true) override

void	revertToKernelAddressSpace () override

bool	memIsInKernelHeap (void *address) override

bool	memIsInHeap (void *address) override

void *	getEndOfHeap () override

uintptr_t	getKernelStart () const override

uintptr_t	getUserStart () const override

uintptr_t	getUserReservedStart () const override

uintptr_t	getDynamicLinkerAddress () const override

uintptr_t	getKernelHeapStart () const override

uintptr_t	getKernelHeapEnd () const override

uintptr_t	getKernelCacheStart () const override

uintptr_t	getKernelCacheEnd () const override

uintptr_t	getKernelEventBlockStart () const override

uintptr_t	getKernelModulesStart () const override

uintptr_t	getKernelModulesEnd () const override

uintptr_t	getGlobalInfoBlock () const override

uintptr_t	getDynamicStart () const override

uintptr_t	getDynamicEnd () const override

physical_uintptr_t	root () const

Public Member Functions inherited from VirtualAddressSpace
RawUserMemory &	rawUserMemory ()

virtual size_t	runtimeMappingPages (uintptr_t, size_t)

MemoryLockAccount *	memoryLockAccount () const

void	setMemoryLockAccount (MemoryLockAccount *account)

UserMemoryPolicy *	userMemoryPolicy () const

void	setUserMemoryPolicy (UserMemoryPolicy *policy)

virtual void *	expandHeap (ssize_t incr, size_t flags)

virtual bool	mapHuge (physical_uintptr_t physAddress, void *virtualAddress, size_t count, size_t flags)

virtual ResidentCopyStatus	copyResidentUserPage (uintptr_t userAddress, void *kernelBuffer, size_t bytes, bool write)

virtual bool	tryWriteUser32 (uintptr_t address, uint32_t value)

virtual bool	tryReadUser32 (uintptr_t address, uint32_t &value)

virtual bool	tryReadUserPointer (uintptr_t address, uintptr_t &value)

virtual bool	tryCompareExchangeUser32 (uintptr_t address, uint32_t &expected, uint32_t desired, bool &exchanged)

virtual RemapStatus	prepareRemap (const PageRemapRequest &request, UniquePointer< PreparedPageRemap > &plan)

virtual void	setRemapPreparationFailureForTest (ssize_t allocations)

virtual	~VirtualAddressSpace ()

void	setHeap (void heap, void heapEnd)

virtual uintptr_t	getKernelVirtualPagestack () const

virtual uintptr_t	getKernelVirtualPagestackAdd1 () const

virtual uintptr_t	getKernelVirtualPagestackAdd2 () const

Private Member Functions
	Arm64VirtualAddressSpace (bool kernel)

uint64_t *	findEntry (uintptr_t address, bool create, size_t *newTables=nullptr)

uint64_t *	findExistingEntry (uintptr_t address, size_t *level=nullptr) const

bool	cloneTable (uint64_t destination, const uint64_t source, size_t level, bool copyOnWrite)

void	freeTable (uint64_t *table, size_t level, bool freeLeaves)

Static Private Member Functions
static uint64_t	pageDescriptor (physical_uintptr_t physical, size_t flags)

static size_t	descriptorFlags (uint64_t descriptor)

Private Attributes
physical_uintptr_t	m_Root

uintptr_t	m_StackTop

Spinlock	m_Lock

Static Private Attributes
static Arm64VirtualAddressSpace	m_KernelSpace

Friends
class	ProcessorBase

VirtualAddressSpace &	VirtualAddressSpace::getKernelAddressSpace ()

VirtualAddressSpace *	VirtualAddressSpace::create ()

Additional Inherited Members
Public Types inherited from VirtualAddressSpace
enum class	ResidentCopyStatus { Success , Inaccessible , Unsupported }

enum class	RemapStatus { Success , InvalidRange , Unsupported , NoMemory , Retry }

using	RemapAdmission = bool()(void context)

Static Public Member Functions inherited from VirtualAddressSpace
static EXPORTED_PUBLIC VirtualAddressSpace &	getKernelAddressSpace ()

static VirtualAddressSpace *	create ()

Public Attributes inherited from VirtualAddressSpace
void *	m_Heap

void *	m_HeapEnd

Static Public Attributes inherited from VirtualAddressSpace
static const size_t	KernelMode = 0x01

static const size_t	Write = 0x02

static const size_t	Execute = 0x04

static const size_t	WriteThrough = 0x08

static const size_t	CacheDisable = 0x10

static const size_t	CopyOnWrite = 0x20

static const size_t	Swapped = 0x40

static const size_t	Shared = 0x200

static const size_t	WriteCombine = 0x400

static const size_t	Accessed = 0x800

static const size_t	Dirty = 0x1000

static const size_t	ClearDirty = 0x2000

static const size_t	NoAccess = 0x4000

static const size_t	WriteProtected = 0x8000

static const size_t	Borrowed = 0x10000

static const size_t	RuntimeMapping = 0x20000

static constexpr size_t	MaximumRemapPages = 65536

static physical_uintptr_t	m_ZeroPage = 0

Protected Member Functions inherited from VirtualAddressSpace
bool	admitRawMemoryChange (const PreparedMemoryLock &plan, bool privileged, MemoryLockCharge &charge) const

void	commitRawMemoryChange (PreparedMemoryLock &plan, MemoryLockCharge charge)

bool	prepareZeroPage ()

	VirtualAddressSpace (void *Heap)

Protected Attributes inherited from VirtualAddressSpace
uint64_t	m_HeapRegionId = 0

Detailed Description

Definition at line 28 of file kernel/core/processor/arm64/VirtualAddressSpace.h.

Constructor & Destructor Documentation

◆ ~Arm64VirtualAddressSpace()

Arm64VirtualAddressSpace::~Arm64VirtualAddressSpace ( )

override

Definition at line 58 of file arm64/VirtualAddressSpace.cc.

◆ Arm64VirtualAddressSpace()

Arm64VirtualAddressSpace::Arm64VirtualAddressSpace ( bool kernel )

explicitprivate

Definition at line 47 of file arm64/VirtualAddressSpace.cc.

Member Function Documentation

◆ allocateStack() [1/2]

VirtualAddressSpace::Stack * Arm64VirtualAddressSpace::allocateStack ( )

overridevirtual

Allocates a single stack for a thread. Will use the default kernel thread size.

Implements VirtualAddressSpace.

Definition at line 347 of file arm64/VirtualAddressSpace.cc.

References allocateStack().

Referenced by allocateStack(), and allocateStack().

Here is the caller graph for this function:

◆ allocateStack() [2/2]

VirtualAddressSpace::Stack * Arm64VirtualAddressSpace::allocateStack ( size_t stackSz )

overridevirtual

Allocates a single stack of the given size for a thread.

Reimplemented from VirtualAddressSpace.

Definition at line 351 of file arm64/VirtualAddressSpace.cc.

References allocateStack(), detachMapping(), Arm64PhysicalMemoryManager::freePage(), VirtualAddressSpace::KernelMode, map(), and VirtualAddressSpace::Write.

◆ clone()

VirtualAddressSpace * Arm64VirtualAddressSpace::clone ( bool copyOnWrite = true )

overridevirtual

Clone this VirtualAddressSpace. That means that we copy-on-write-map the application * image. *

Parameters

[in] copyOnWrite whether the new address space copies pages on write, or shares them with the parent address space.

Returns: pointer to the new VirtualAddressSpace

Implements VirtualAddressSpace.

Definition at line 459 of file arm64/VirtualAddressSpace.cc.

References clone(), VirtualAddressSpace::m_Heap, and VirtualAddressSpace::m_HeapEnd.

Referenced by clone().

Here is the caller graph for this function:

◆ cloneTable()

bool Arm64VirtualAddressSpace::cloneTable	(	uint64_t *	destination,
		const uint64_t *	source,
		size_t	level,
		bool	copyOnWrite
	)

private

Definition at line 402 of file arm64/VirtualAddressSpace.cc.

◆ descriptorFlags()

size_t Arm64VirtualAddressSpace::descriptorFlags ( uint64_t descriptor )

staticprivate

Definition at line 175 of file arm64/VirtualAddressSpace.cc.

◆ detachMapping()

bool Arm64VirtualAddressSpace::detachMapping	(	void *	virtualAddress,
		physical_uintptr_t &	physical,
		size_t &	flags,
		size_t	requiredFlags = `0`
	)

overridevirtual

Atomically snapshot and detach a resident mapping. A flags mismatch leaves the mapping intact, but still returns its physical address and flags.

Reimplemented from VirtualAddressSpace.

Definition at line 287 of file arm64/VirtualAddressSpace.cc.

References ProcessorBase::invalidate().

Referenced by allocateStack(), freeStack(), handleCopyOnWriteFault(), and unmap().

Here is the caller graph for this function:

◆ findEntry()

uint64_t * Arm64VirtualAddressSpace::findEntry	(	uintptr_t	address,
		bool	create,
		size_t *	newTables = `nullptr`
	)

private

Definition at line 91 of file arm64/VirtualAddressSpace.cc.

◆ findExistingEntry()

uint64_t * Arm64VirtualAddressSpace::findExistingEntry	(	uintptr_t	address,
		size_t *	level = `nullptr`
	)		const

private

Definition at line 70 of file arm64/VirtualAddressSpace.cc.

◆ freeStack()

void Arm64VirtualAddressSpace::freeStack ( Stack * pStack )

overridevirtual

Frees a stack allocated with allocateStack.

Implements VirtualAddressSpace.

Definition at line 386 of file arm64/VirtualAddressSpace.cc.

References detachMapping(), and Arm64PhysicalMemoryManager::freePage().

◆ freeTable()

void Arm64VirtualAddressSpace::freeTable	(	uint64_t *	table,
		size_t	level,
		bool	freeLeaves
	)

private

Definition at line 442 of file arm64/VirtualAddressSpace.cc.

◆ getDynamicEnd()

uintptr_t Arm64VirtualAddressSpace::getDynamicEnd ( ) const

inlineoverridevirtual

Gets address of the end of the dynamic memory mapping area.

Reimplemented from VirtualAddressSpace.

Definition at line 95 of file kernel/core/processor/arm64/VirtualAddressSpace.h.

◆ getDynamicLinkerAddress()

uintptr_t Arm64VirtualAddressSpace::getDynamicLinkerAddress ( ) const

inlineoverridevirtual

Gets address of the dynamic linker in the address space.

Implements VirtualAddressSpace.

Definition at line 65 of file kernel/core/processor/arm64/VirtualAddressSpace.h.

◆ getDynamicStart()

uintptr_t Arm64VirtualAddressSpace::getDynamicStart ( ) const

inlineoverridevirtual

Gets address of the start of the dynamic memory mapping area. This is an area in which memory mappings can be created for userspace applications. Systems that do not have room in the address space for this may return zero.

Reimplemented from VirtualAddressSpace.

Definition at line 92 of file kernel/core/processor/arm64/VirtualAddressSpace.h.

◆ getEndOfHeap()

void * Arm64VirtualAddressSpace::getEndOfHeap ( )

overridevirtual

Gets a pointer to the byte after the end of the heap.

Implements VirtualAddressSpace.

Definition at line 499 of file arm64/VirtualAddressSpace.cc.

◆ getGlobalInfoBlock()

uintptr_t Arm64VirtualAddressSpace::getGlobalInfoBlock ( ) const

inlineoverridevirtual

Gets address of the global info block location.

Reimplemented from VirtualAddressSpace.

Definition at line 89 of file kernel/core/processor/arm64/VirtualAddressSpace.h.

◆ getKernelCacheEnd()

uintptr_t Arm64VirtualAddressSpace::getKernelCacheEnd ( ) const

inlineoverridevirtual

Gets address of the end of the kernel's cache region.

Implements VirtualAddressSpace.

Definition at line 77 of file kernel/core/processor/arm64/VirtualAddressSpace.h.

◆ getKernelCacheStart()

uintptr_t Arm64VirtualAddressSpace::getKernelCacheStart ( ) const

inlineoverridevirtual

Gets address of the start of the kernel's cache region.

Implements VirtualAddressSpace.

Definition at line 74 of file kernel/core/processor/arm64/VirtualAddressSpace.h.

◆ getKernelEventBlockStart()

uintptr_t Arm64VirtualAddressSpace::getKernelEventBlockStart ( ) const

inlineoverridevirtual

Gets address of the start of the kernel's event handling block.

Implements VirtualAddressSpace.

Definition at line 80 of file kernel/core/processor/arm64/VirtualAddressSpace.h.

◆ getKernelHeapEnd()

uintptr_t Arm64VirtualAddressSpace::getKernelHeapEnd ( ) const

inlineoverridevirtual

Gets address of the end of the kernel's heap region.

Implements VirtualAddressSpace.

Definition at line 71 of file kernel/core/processor/arm64/VirtualAddressSpace.h.

◆ getKernelHeapStart()

uintptr_t Arm64VirtualAddressSpace::getKernelHeapStart ( ) const

inlineoverridevirtual

Gets address of the start of the kernel's heap region.

Implements VirtualAddressSpace.

Definition at line 68 of file kernel/core/processor/arm64/VirtualAddressSpace.h.

◆ getKernelModulesEnd()

uintptr_t Arm64VirtualAddressSpace::getKernelModulesEnd ( ) const

inlineoverridevirtual

Gets address of the end of the kernel's module region.

Implements VirtualAddressSpace.

Definition at line 86 of file kernel/core/processor/arm64/VirtualAddressSpace.h.

◆ getKernelModulesStart()

uintptr_t Arm64VirtualAddressSpace::getKernelModulesStart ( ) const

inlineoverridevirtual

Gets address of the start of the kernel's module region.

Implements VirtualAddressSpace.

Definition at line 83 of file kernel/core/processor/arm64/VirtualAddressSpace.h.

◆ getKernelStart()

uintptr_t Arm64VirtualAddressSpace::getKernelStart ( ) const

inlineoverridevirtual

Gets start address of the kernel in the address space.

Implements VirtualAddressSpace.

Definition at line 56 of file kernel/core/processor/arm64/VirtualAddressSpace.h.

◆ getMapping()

bool Arm64VirtualAddressSpace::getMapping	(	void *	virtualAddress,
		physical_uintptr_t &	physicalAddress,
		size_t &	flags
	)

overridevirtual

Query the physical address and flags of a mapped or swapped-out page. Returns false without modifying the outputs when no mapping exists. The result does not pin the page; callers must protect its lifetime while using the returned physical address.

Implements VirtualAddressSpace.

Definition at line 252 of file arm64/VirtualAddressSpace.cc.

Referenced by handleCopyOnWriteFault().

Here is the caller graph for this function:

◆ getUserReservedStart()

uintptr_t Arm64VirtualAddressSpace::getUserReservedStart ( ) const

inlineoverridevirtual

Gets start address of reserved areas of the userpace address space.

Implements VirtualAddressSpace.

Definition at line 62 of file kernel/core/processor/arm64/VirtualAddressSpace.h.

◆ getUserStart()

uintptr_t Arm64VirtualAddressSpace::getUserStart ( ) const

inlineoverridevirtual

Gets start address of the region usable and cloneable for userspace.

Implements VirtualAddressSpace.

Definition at line 59 of file kernel/core/processor/arm64/VirtualAddressSpace.h.

◆ handleCopyOnWriteFault()

bool Arm64VirtualAddressSpace::handleCopyOnWriteFault	(	void *	virtualAddress,
		bool	userMode
	)

overridevirtual

Resolve a write-protection fault on a copy-on-write mapping.

The replacement page is initialised before it becomes visible. Concurrent callers revalidate the mapping and treat an already-writable replacement as a successfully handled stale fault when the faulting privilege level may access it.

Parameters

[in] userMode true when the faulting access came from user mode.

Implements VirtualAddressSpace.

Definition at line 323 of file arm64/VirtualAddressSpace.cc.

References VirtualAddressSpace::Borrowed, VirtualAddressSpace::CopyOnWrite, detachMapping(), Arm64PhysicalMemoryManager::freePage(), getMapping(), VirtualAddressSpace::KernelMode, map(), VirtualAddressSpace::NoAccess, VirtualAddressSpace::Shared, VirtualAddressSpace::Swapped, VirtualAddressSpace::Write, and VirtualAddressSpace::WriteProtected.

◆ isAddressValid()

bool Arm64VirtualAddressSpace::isAddressValid ( void * virtualAddress )

overridevirtual

Is a particular virtual address valid?

Parameters

[in] virtualAddress the virtual address to check

Returns: true, if the address is valid, false otherwise

Implements VirtualAddressSpace.

Definition at line 65 of file arm64/VirtualAddressSpace.cc.

Referenced by map().

Here is the caller graph for this function:

◆ isMapped()

bool Arm64VirtualAddressSpace::isMapped ( void * virtualAddress )

overridevirtual

Checks whether a mapping the the specific virtual address exists. Pages marked as swapped out are not considered mapped; retained NoAccess pages are.

Note: This function must be valid on all the valid addresses within the virtual address space.

Parameters

[in] virtualAddress the virtual address

Returns: true, if a mapping exists, false otherwise

Implements VirtualAddressSpace.

Definition at line 213 of file arm64/VirtualAddressSpace.cc.

◆ map()

bool Arm64VirtualAddressSpace::map	(	physical_uintptr_t	physicalAddress,
		void *	virtualAddress,
		size_t	flags
	)

overridevirtual

Map a specific physical page (of size PhysicalMemoryManager::getPageSize()) at a specific location into the virtual address space.

Note: This function must also work on pages marked as swapped out.

Parameters

[in]	physicalAddress	the address of the physical page that should be mapped into the virtual address space.
[in]	virtualAddress	the virtual address at which the page apears within the virtual address space.
[in]	flags	flags that describe which accesses should be allowed on the page.