MemoryMappedFile.h

Go to the documentation of this file.

/*
 * Copyright (c) 2008-2014, Pedigree Developers
 *
 * Please see the CONTRIB file in the root of the source tree for a full
 * list of contributors.
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */
 
#ifndef MEMORY_MAPPED_FILE_H
#define MEMORY_MAPPED_FILE_H
#include "pedigree/kernel/Spinlock.h"
#include "pedigree/kernel/compiler.h"
#include "pedigree/kernel/process/FilesystemContext.h"
#include "pedigree/kernel/process/MemoryPressureManager.h"
#include "pedigree/kernel/process/Mutex.h"
#include "pedigree/kernel/process/TerminationDeferral.h"
#include "pedigree/kernel/process/Uninterruptible.h"
#include "pedigree/kernel/processor/PageFaultHandler.h"
#include "pedigree/kernel/processor/UserMemoryPolicy.h"
#include "pedigree/kernel/processor/VirtualAddressSpace.h"
#include "pedigree/kernel/processor/state_forward.h"
#include "pedigree/kernel/processor/types.h"
#include "pedigree/kernel/utilities/Cache.h"
#include "pedigree/kernel/utilities/List.h"
#include "pedigree/kernel/utilities/SharedPointer.h"
#include "pedigree/kernel/utilities/String.h"
#include "pedigree/kernel/utilities/Tree.h"
#include "pedigree/kernel/utilities/new"
 
#include <config.h>
 
#include "MappingList.h"
#include "SwapStore.h"
 
class File;
class Process;
class VirtualAddressSpace;
class VfsUserMemoryPolicy;
using FileResidencyAccess = bool (*)(File*, void*);
 
struct EXPORTED_PUBLIC FileMappingOrigin {
  uint64_t openIdentity = 0;
  bool writableOpen = false;
  FilesystemPathRef openingPath{};
  SharedPointer<FilesystemWriteLease> writeLease{};
};
 
class MappingAttachment {
 public:
  virtual ~MappingAttachment() {}
  virtual uintptr_t baseAddress() const = 0;
  virtual void relocate(uintptr_t newBase) = 0;
  virtual SharedPointer<MappingAttachment> clone(Process* target) = 0;
};
 
class MemoryMappedObject {
  friend class MemoryMapManager;
 
 private:
  MemoryMappedObject();
 
 public:
  typedef int Permissions;
 
  static const int None = 0x0;
  static const int Read = 0x1;
  static const int Write = 0x2;
  static const int Exec = 0x4;
 
  MemoryMappedObject(uintptr_t address, bool bCopyOnWrite, size_t length, Permissions perms,
                     Permissions maximumPerms = Read | Write | Exec)
      : m_bCopyOnWrite(bCopyOnWrite),
        m_Address(address),
        m_Length(length),
        m_Permissions(perms),
        m_MaximumPermissions(maximumPerms),
        m_Attachment(),
        m_OwnerProcess(nullptr),
        m_OwnsMappings(true),
        m_LockMode(MemoryLockMode::None) {}
 
  virtual ~MemoryMappedObject();
 
  virtual MemoryMappedObject* clone() = 0;
 
  virtual MemoryMappedObject* split(uintptr_t at) = 0;
 
  virtual MemoryMappedObject* stageSlice(uintptr_t source, size_t sourceLength,
                                         uintptr_t destination, size_t destinationLength) = 0;
  void setMappingOwnership(bool ownsMappings) {
    m_OwnsMappings = ownsMappings;
  }
  virtual void releaseDetachedPage(uintptr_t oldAddress,
                                   const VirtualAddressSpace::DetachedPage& page) = 0;
  virtual void discardRange(VirtualAddressSpace& space, uintptr_t base, size_t length) = 0;
  virtual File* backingFile() const {
    return nullptr;
  }
  virtual bool backingRangeValid(uintptr_t source, size_t length) const {
    return true;
  }
  virtual bool resident(VirtualAddressSpace& space, uintptr_t address, FileResidencyAccess access,
                        void* credentials);
 
  virtual bool remove(size_t length) = 0;
 
  virtual void setPermissions(Permissions perms) = 0;
 
  virtual bool preparePermissions(uintptr_t base, size_t length, Permissions perms) {
    return true;
  }
 
  virtual bool sync(uintptr_t at, bool async) {
    return true;
  }
 
  virtual bool syncRange(uintptr_t at, size_t length, bool async);
 
  virtual void invalidate(uintptr_t at) {}
 
  virtual bool sharedBacking(uintptr_t at, uintptr_t& identity, size_t& offset) const {
    return false;
  }
 
  virtual bool usesBacking(uintptr_t identity) const {
    return false;
  }
  virtual bool beyondBackingEnd(uintptr_t at) const {
    return false;
  }
  virtual void discardFilePages(VirtualAddressSpace& space, size_t end) {}
 
  virtual void unmap() = 0;
 
  virtual bool trap(VirtualAddressSpace& space, uintptr_t address, bool bWrite,
                    PopulationStatus* population = nullptr) = 0;
  virtual PopulationStatus prepareResidentAccess(VirtualAddressSpace& space, uintptr_t address) {
    return PopulationStatus::Success;
  }
  PopulationStatus populatePage(VirtualAddressSpace& space, uintptr_t address);
  MemoryLockMode lockMode() const {
    return m_LockMode;
  }
 
  virtual bool supportsPageOut(VirtualAddressSpace& space, uintptr_t address) {
    return false;
  }
  virtual SwapStatus pageOutAt(VirtualAddressSpace& space, uintptr_t address, bool& released) {
    return SwapStatus::Unsupported;
  }
  virtual bool reclaimAnonymousPage(VirtualAddressSpace& space) {
    return false;
  }
  virtual SwapStatus restoreSwapPages(VirtualAddressSpace& space) {
    return SwapStatus::Success;
  }
  virtual bool compact() {
    return false;
  }
 
  bool matches(uintptr_t address) {
    return (m_Address <= address) && (address < (m_Address + m_Length));
  }
 
  uintptr_t address() const {
    return m_Address;
  }
 
  size_t length() const {
    return m_Length;
  }
 
  Permissions permissions() const {
    return m_Permissions;
  }
 
  Permissions maximumPermissions() const {
    return m_MaximumPermissions;
  }
 
 protected:
  bool m_bCopyOnWrite;
 
  uintptr_t m_Address;
 
  size_t m_Length;
 
  Permissions m_Permissions;
  Permissions m_MaximumPermissions;
  SharedPointer<MappingAttachment> m_Attachment;
  // The mapping gate protects registration; paging additionally admits a
  // short scheduler lease before dereferencing this process identity.
  Process* m_OwnerProcess;
  bool m_OwnsMappings;
  MemoryLockMode m_LockMode;
};
 
class EXPORTED_PUBLIC AnonymousMemoryMap : public MemoryMappedObject {
 public:
  AnonymousMemoryMap(uintptr_t address, size_t length, Permissions perms);
 
  virtual ~AnonymousMemoryMap() override;
#if HOSTED && PEDIGREE_HOSTED_SMOKE_TESTS
  static void setCloneFailureForTest(ssize_t after);
#endif
 
  virtual MemoryMappedObject* clone() override;
  virtual MemoryMappedObject* split(uintptr_t at) override;
  MemoryMappedObject* stageSlice(uintptr_t source, size_t sourceLength, uintptr_t destination,
                                 size_t destinationLength) override;
  void releaseDetachedPage(uintptr_t oldAddress,
                           const VirtualAddressSpace::DetachedPage& page) override;
  void discardRange(VirtualAddressSpace& space, uintptr_t base, size_t length) override;
  virtual bool remove(size_t length) override;
 
  virtual void setPermissions(MemoryMappedObject::Permissions perms) override;
 
  virtual void unmap() override;
 
  virtual bool trap(VirtualAddressSpace& space, uintptr_t address, bool bWrite,
                    PopulationStatus* population = nullptr) override;
  bool supportsPageOut(VirtualAddressSpace& space, uintptr_t address) override;
  SwapStatus pageOutAt(VirtualAddressSpace& space, uintptr_t address, bool& released) override;
  bool reclaimAnonymousPage(VirtualAddressSpace& space) override {
    return pageOut(space);
  }
  SwapStatus restoreSwapPages(VirtualAddressSpace& space) override {
    return restoreAll(space);
  }
  PopulationStatus prepareResidentAccess(VirtualAddressSpace& space, uintptr_t address) override;
 
 private:
  friend class MemoryMapManager;
  struct Page {
    uintptr_t address = 0;
    SwapReference slot;
    bool pagingBlocked = false;
  };
  bool pageOut(VirtualAddressSpace& space);
  SwapStatus restorePage(VirtualAddressSpace& space, Page& page);
  SwapStatus restoreAll(VirtualAddressSpace& space);
  static physical_uintptr_t m_Zero;
  static bool initialisePhysicalPage(physical_uintptr_t physical);
 
  void unmapUnlocked();
 
  // Faults must not scan every previously populated page in a large mapping.
  Tree<uintptr_t, Page> m_Mappings;
};
 
class MemoryMappedFile : public MemoryMappedObject {
  friend class MemoryMapManager;
 
 public:
  MemoryMappedFile(
      uintptr_t address, size_t length, size_t offset, File* backing, bool bCopyOnWrite,
      Permissions perms, Permissions maximumPerms = Read | Write | Exec,
      const SharedPointer<MappingAttachment>& attachment = SharedPointer<MappingAttachment>(),
      const FileMappingOrigin& origin = {}, bool executableUse = false);
 
  virtual ~MemoryMappedFile() override;
 
  virtual MemoryMappedObject* clone() override;
  virtual MemoryMappedObject* split(uintptr_t at) override;
  MemoryMappedObject* stageSlice(uintptr_t source, size_t sourceLength, uintptr_t destination,
                                 size_t destinationLength) override;
  void releaseDetachedPage(uintptr_t oldAddress,
                           const VirtualAddressSpace::DetachedPage& page) override;
  void discardRange(VirtualAddressSpace& space, uintptr_t base, size_t length) override;
  File* backingFile() const override {
    return m_pBacking;
  }
  bool backingRangeValid(uintptr_t source, size_t length) const override;
  bool resident(VirtualAddressSpace& space, uintptr_t address, FileResidencyAccess access,
                void* credentials) override;
  virtual bool remove(size_t length) override;
 
  virtual void setPermissions(MemoryMappedObject::Permissions perms) override;
 
  virtual bool sync(uintptr_t at, bool async) override;
  bool syncRange(uintptr_t at, size_t length, bool async) override;
  virtual void invalidate(uintptr_t at) override;
  virtual bool sharedBacking(uintptr_t at, uintptr_t& identity, size_t& offset) const override;
  virtual bool usesBacking(uintptr_t identity) const override;
  virtual bool beyondBackingEnd(uintptr_t at) const override;
  virtual void discardFilePages(VirtualAddressSpace& space, size_t end) override;
  virtual bool preparePermissions(uintptr_t base, size_t length, Permissions perms) override;
 
  virtual void unmap() override;
 
  virtual bool trap(VirtualAddressSpace& space, uintptr_t address, bool bWrite,
                    PopulationStatus* population = nullptr) override;
 
  virtual bool compact() override;
 
 private:
  void unmapUnlocked();
 
  physical_uintptr_t getBackingPage(size_t fileOffset, size_t mappingBytes);
 
  void releaseDetachedPageUnlocked(uintptr_t oldAddress,
                                   const VirtualAddressSpace::DetachedPage& page);
 
  void trackMapping(uintptr_t, physical_uintptr_t);
 
  void untrackMapping(uintptr_t);
 
  physical_uintptr_t getMapping(uintptr_t);
 
  size_t getMappingCount();
 
  void clearMappings();
 
  File* m_pBacking;
 
  size_t m_Offset;
 
  size_t m_ReadAheadEnd = ~size_t{0};
  size_t m_ReadAheadPages = 4;
 
  Tree<uintptr_t, physical_uintptr_t> m_Mappings;
  FileMappingOrigin m_Origin;
 
  Mutex m_Lock;
 
  bool m_bVfsLease;
  bool m_ExecutableUse;
  bool m_SharedWriteUse;
  bool m_UseAdmitted;
  bool m_MountAdmitted;
};
 
class EXPORTED_PUBLIC MemoryMapManager : public MemoryTrapHandler, public MemoryPressureHandler {
  friend class PosixSubsystem;
  friend class VfsUserMemoryPolicy;
 
 public:
  enum class Placement {
    Hint,
    FixedReplace,
    FixedNoReplace,
  };
 
  enum class MapStatus {
    Success,
    NoMemory,
    AddressInUse,
    PolicyDenied,
    TextBusy,
    LockLimit,
  };
 
  enum class VmStatus { Success, InvalidRange, Unmapped, Unsupported, NoMemory, LockLimit };
  enum class FileRemapStatus {
    Success,
    InvalidRange,
    Unsupported,
    NoMemory,
    PolicyDenied,
    LockLimit,
    PermissionDenied
  };
  FileRemapStatus remapFilePages(uintptr_t base, size_t length, size_t byteOffset, bool nonblock);
  struct RemapRequest {
    uintptr_t source, destination;
    size_t oldLength, newLength;
    bool mayMove, fixed;
  };
  VmStatus remap(const RemapRequest& request, uintptr_t& result);
  VmStatus residency(uintptr_t base, size_t length, unsigned char* kernelVector,
                     FileResidencyAccess access, void* credentials);
  VmStatus discard(uintptr_t base, size_t length);
 
  enum class UserPageCopyStatus { Success, Inaccessible, NoMemory, IoError, Unsupported };
  UserPageCopyStatus copyUserPage(VirtualAddressSpace& space, uintptr_t userAddress,
                                  void* kernelBuffer, size_t bytes, bool write);
 
  PopulationStatus populateMemory(VirtualAddressSpace& space, uintptr_t base, size_t length);
  void bindMemoryLockPolicy(VirtualAddressSpace& space);
  MemoryLockStatus lockMemory(VirtualAddressSpace& space, uintptr_t base, size_t length,
                              MemoryLockMode mode, bool privileged);
  MemoryLockStatus lockAllMemory(VirtualAddressSpace& space, bool current,
                                 MemoryLockMode currentMode, MemoryLockMode futureMode,
                                 bool privileged);
  bool hasLockedMemory(VirtualAddressSpace& space, uintptr_t base, size_t length);
 
  static MemoryMapManager& instance() {
    return m_Instance;
  }
 
#if HOSTED && PEDIGREE_HOSTED_SMOKE_TESTS
  void acquireLifecycleGateForHostedTest();
  void releaseLifecycleGateForHostedTest();
  bool trapForHostedTest(uintptr_t address, bool bIsWrite, bool bWasPresent);
  const void* lifecycleGateAddressForHostedTest() const {
    return static_cast<const Semaphore*>(&m_LifecycleLock);
  }
#endif
 
  MemoryMappedObject* mapFile(File* pFile, uintptr_t& address, size_t length,
                              MemoryMappedObject::Permissions perms, size_t offset = 0,
                              bool bCopyOnWrite = true);
 
  bool hasSharedWriteCapability(File* backing);
 
  MemoryMappedObject* mapFile(
      File* pFile, uintptr_t& address, size_t length, MemoryMappedObject::Permissions perms,
      size_t offset, bool bCopyOnWrite, Placement placement, MapStatus* status,
      MemoryMappedObject::Permissions maximumPerms = MemoryMappedObject::Read |
                                                     MemoryMappedObject::Write |
                                                     MemoryMappedObject::Exec,
      const SharedPointer<MappingAttachment>& attachment = SharedPointer<MappingAttachment>(),
      MemoryLockMode requestedLock = MemoryLockMode::None, const FileMappingOrigin& origin = {});
 
  MemoryMappedObject* mapAnon(uintptr_t& address, size_t length,
                              MemoryMappedObject::Permissions perms);
 
  MemoryMappedObject* mapAnon(uintptr_t& address, size_t length,
                              MemoryMappedObject::Permissions perms, Placement placement,
                              MapStatus* status,
                              MemoryLockMode requestedLock = MemoryLockMode::None);
 
  bool clone(Process* pTarget);
  SwapStatus pageOutRange(uintptr_t base, size_t length);
  SwapStatus activateSwap(uint32_t endpoint);
  SwapStatus deactivateSwap(uint32_t endpoint);
  SwapSnapshot swapSnapshot();
  bool operationOwnedByCurrentExecution();
 
  size_t remove(uintptr_t base, size_t length);
 
  size_t removeAndRelease(uintptr_t base, size_t length, VmStatus* status = nullptr);
 
  SharedPointer<MappingAttachment> findAttachment(uintptr_t base);
 
  size_t removeAttachment(const SharedPointer<MappingAttachment>& attachment);
 
  enum class ProtectStatus {
    Success,
    Unmapped,
    AccessDenied,
    InvalidRange,
    Unsupported,
    NoMemory,
    TextBusy
  };
 
  size_t setPermissions(uintptr_t base, size_t length, MemoryMappedObject::Permissions perms,
                        ProtectStatus* status = nullptr);
 
  bool contains(uintptr_t base, size_t length);
 
  bool allows(uintptr_t base, size_t length, MemoryMappedObject::Permissions permissions);
 
  bool sharedBacking(Process* process, uintptr_t address, uintptr_t& identity, size_t& offset);
 
  bool faultIn(uintptr_t address, bool write);
  bool faultInRange(uintptr_t address, size_t length, bool write);
  bool writableAnonymousRange(uintptr_t address, size_t length);
 
  enum class FaultResolution { Unhandled, Resolved, BackingFault };
  FaultResolution resolveUserFault(uintptr_t address, bool write, bool wasPresent, bool execute);
 
  enum class ResizeStatus { Ready, NoMemory, Unsupported, Invalid };
  class PreparedFileResize {
   public:
    ~PreparedFileResize();
    const Cache::DiscardReference* loans() const;
    size_t loanCount() const;
    size_t totalLoans() const;
    void commit();
 
   private:
    friend class MemoryMapManager;
    struct Data;
    explicit PreparedFileResize(Data* data);
    NOT_COPYABLE_OR_ASSIGNABLE(PreparedFileResize);
    UniquePointer<Data> m_Data;
  };
 
  ResizeStatus prepareFileResize(File* file, size_t oldSize, size_t newSize,
                                 UniquePointer<PreparedFileResize>& result);
 
  bool sync(uintptr_t base, size_t length, bool async, int* error = nullptr);
 
  void invalidate(uintptr_t base, size_t length);
 
  void unmap(MemoryMappedObject* pObj);
 
  void unmapAll();
 
  virtual bool trap(InterruptState& state, uintptr_t address, bool bIsWrite, bool bWasPresent);
 
  virtual bool compact();
 
  virtual const char* getMemoryPressureDescription() {
    return "Unmap safe pages from memory mapped files.";
  }
 
 protected:
  void unmapAllUnlocked();
 
  void acquireLock();
 
  void releaseLock();
 
 public:
  class OperationGuard {
   public:
    explicit OperationGuard(MemoryMapManager& manager, bool tryOnly = false);
    ~OperationGuard();
 
    explicit operator bool() const {
      return m_Acquired;
    }
 
   private:
    NOT_COPYABLE_OR_ASSIGNABLE(OperationGuard);
 
    Uninterruptible m_EventDeferral;
    MemoryMapManager& m_Manager;
    bool m_Acquired;
  };
 
 private:
  MemoryMapManager();
  ~MemoryMapManager();
 
  class LockPlan;
  MemoryLockStatus prepareManagedLocks(VirtualAddressSpace& space, uintptr_t base, size_t length,
                                       MemoryLockMode mode, bool all,
                                       UniquePointer<PreparedMemoryLock>& result);
  MemoryMappedObject* publishMapping(File* file, uintptr_t& address, size_t length,
                                     MemoryMappedObject::Permissions perms, size_t offset,
                                     bool copyOnWrite, Placement placement, MapStatus* status,
                                     MemoryMappedObject::Permissions maximumPerms,
                                     const SharedPointer<MappingAttachment>& attachment,
                                     MemoryLockMode requestedLock,
                                     const FileMappingOrigin& origin = {});
  void retireLockedPages(VirtualAddressSpace& space, size_t pages);
 
  void enterOperation();
  bool tryEnterOperation();
  void leaveOperation();
 
  size_t removeInternal(uintptr_t base, size_t length, bool releaseReservations,
                        VmStatus* status = nullptr);
  void releaseReservation(Process* process, VirtualAddressSpace& addressSpace, uintptr_t base,
                          size_t length);
  bool faultInUnlocked(uintptr_t address, bool write, MemoryMappedObject*& selected,
                       bool residentAnonymousOnly = false);
  bool accessRange(uintptr_t address, size_t length, bool write, bool residentAnonymousOnly);
  bool handleTrapUnlocked(uintptr_t address, bool bIsWrite, bool bWasPresent, bool execute,
                          MemoryMappedObject* selected);
  bool handleTrap(uintptr_t address, bool bIsWrite, bool bWasPresent, bool execute = false,
                  MemoryMappedObject* selected = nullptr);
 
  enum Ops {
    Sync,
    Invalidate,
  };
 
  bool op(Ops what, uintptr_t base, size_t length, bool async);
 
  static MemoryMapManager m_Instance;
 
  typedef MappingList<MemoryMappedObject> MmObjectList;
 
  Tree<VirtualAddressSpace*, MmObjectList*> m_MmObjectLists;
 
  Spinlock m_Lock;
 
  Mutex m_LifecycleLock;
 
  Spinlock m_LifecycleStateLock;
 
  void* m_pLifecycleOwner;
  size_t m_LifecycleDepth;
};
 
#endif