hosted/PhysicalMemoryManager.cc

/*
 * Copyright (c) 2008-2014, Pedigree Developers
 *
 * Please see the CONTRIB file in the root of the source tree for a full
 * list of contributors.
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */
 
#include "PhysicalMemoryManager.h"
#include "pedigree/kernel/LockGuard.h"
#include "pedigree/kernel/Log.h"
#include "pedigree/kernel/Metrics.h"
#include "pedigree/kernel/machine/Trace.h"
#include "pedigree/kernel/panic.h"
#include "pedigree/kernel/processor/MemoryRegion.h"
#include "pedigree/kernel/processor/Processor.h"
#include "pedigree/kernel/utilities/Cache.h"
#include "pedigree/kernel/utilities/assert.h"
#include "pedigree/kernel/utilities/utility.h"
 
#if TRACK_PAGE_ALLOCATIONS
#include "pedigree/kernel/debugger/commands/AllocationCommand.h"
#endif
 
#include "pedigree/kernel/core/SlamAllocator.h"
#include "pedigree/kernel/process/MemoryPressureManager.h"
 
#include "VirtualAddressSpace.h"
 
namespace __pedigree_hosted {};
using namespace __pedigree_hosted;
 
#include <fcntl.h>
#include <stdio.h>
#include <unistd.h>
 
uint32_t g_PageBitmap[16384] = {0};
 
HostedPhysicalMemoryManager* HostedPhysicalMemoryManager::m_Instance = nullptr;
 
static char instanceStorage[sizeof(HostedPhysicalMemoryManager)] = {0};
 
PhysicalMemoryManager& PhysicalMemoryManager::instance() {
  return HostedPhysicalMemoryManager::instance();
}
 
HostedPhysicalMemoryManager& HostedPhysicalMemoryManager::instance() {
  if (!m_Instance) {
    m_Instance = new (reinterpret_cast<void*>(instanceStorage)) HostedPhysicalMemoryManager();
  }
  return *m_Instance;
}
 
physical_uintptr_t HostedPhysicalMemoryManager::allocatePage(size_t pageConstraints) {
  static Atomic<bool> bDidHitWatermark(false);
 
  m_Lock.acquire(true);
 
  physical_uintptr_t ptr;
 
  // Some methods of handling memory pressure require allocating pages, so
  // we need to not end up recursively trying to release the pressure.
  MemoryPressureManager& pressureManager = MemoryPressureManager::instance();
  if (!pressureManager.compactingForCurrentExecution()) {
    if (m_PageStack.freePages() < MemoryPressureManager::getHighWatermark()) {
      // Make sure the compact can trigger frees.
      m_Lock.release();
 
      WARNING_NOLOCK("Memory pressure encountered, performing a compact...");
      if (!pressureManager.compact())
        ERROR_NOLOCK("Compact did not alleviate any memory pressure.");
      else
        NOTICE_NOLOCK("Compact was successful.");
 
      m_Lock.acquire(true);
 
      bDidHitWatermark.compareAndSwap(false, true);
    } else if (bDidHitWatermark.compareAndSwap(true, false)) {
      ERROR_NOLOCK("<pressure was hit, but is no longer being hit>");
    }
  }
 
  ptr = m_PageStack.allocate(0);
  if (!ptr) {
    if (Processor::m_Initialised == 2) {
      Metrics::increment(Metrics::PhysicalPageAllocFailure);
    }
    panic("Out of memory.");
  }
  physical_uintptr_t ptr_bitmap = ptr / getPageSize();
  size_t idx = ptr_bitmap / 32;
  size_t bit = ptr_bitmap % 32;
  if (g_PageBitmap[idx] & (1 << bit)) {
    m_Lock.release();
    FATAL_NOLOCK("PhysicalMemoryManager allocate()d a page twice");
  }
  g_PageBitmap[idx] |= (1 << bit);
  if (Processor::m_Initialised == 2) {
    Metrics::increment(Metrics::PhysicalPageAlloc);
  }
 
  m_Lock.release();
 
#if TRACK_PAGE_ALLOCATIONS
  if (Processor::m_Initialised == 2) {
    if (!g_AllocationCommand.isMallocing()) {
      g_AllocationCommand.allocatePage(ptr);
    }
  }
#endif
 
  return ptr;
}
#if HOSTED && PEDIGREE_HOSTED_SMOKE_TESTS
namespace {
ssize_t tryAllocationFailure = -1;
}
void PhysicalMemoryManager::setTryAllocationFailureForTest(ssize_t after) {
  auto& memory = HostedPhysicalMemoryManager::instance();
  RecursingLockGuard<Spinlock> guard(memory.m_Lock);
  tryAllocationFailure = after;
}
#endif
physical_uintptr_t HostedPhysicalMemoryManager::tryAllocatePage() {
  m_Lock.acquire(true);
#if HOSTED && PEDIGREE_HOSTED_SMOKE_TESTS
  if (!tryAllocationFailure) {
    m_Lock.release();
    if (Processor::m_Initialised == 2) {
      Metrics::increment(Metrics::PhysicalPageAllocFailure);
    }
    return 0;
  }
  if (tryAllocationFailure > 0)
    --tryAllocationFailure;
#endif
  physical_uintptr_t ptr;
  ptr = m_PageStack.allocate(0, false);
  if (!ptr) {
    m_Lock.release();
    if (Processor::m_Initialised == 2) {
      Metrics::increment(Metrics::PhysicalPageAllocFailure);
    }
    return 0;
  }
  physical_uintptr_t ptr_bitmap = ptr / getPageSize();
  size_t idx = ptr_bitmap / 32;
  size_t bit = ptr_bitmap % 32;
  if (g_PageBitmap[idx] & (1 << bit)) {
    m_Lock.release();
    FATAL_NOLOCK("PhysicalMemoryManager allocate()d a page twice");
  }
  g_PageBitmap[idx] |= (1 << bit);
  if (Processor::m_Initialised == 2) {
    Metrics::increment(Metrics::PhysicalPageAlloc);
  }
 
  m_Lock.release();
 
#if TRACK_PAGE_ALLOCATIONS
  if (Processor::m_Initialised == 2) {
    if (!g_AllocationCommand.isMallocing()) {
      g_AllocationCommand.allocatePage(ptr);
    }
  }
#endif
 
  return ptr;
}
 
PhysicalMemoryManager::MemorySnapshot HostedPhysicalMemoryManager::memorySnapshot() const {
  auto& self = *const_cast<HostedPhysicalMemoryManager*>(this);
  RecursingLockGuard<Spinlock> guard(self.m_Lock);
  return {m_PageStack.totalPages(), m_PageStack.freePages(), true};
}
 
bool HostedPhysicalMemoryManager::copyPhysicalPageToBuffer(physical_uintptr_t page, void* buffer) {
  return page && !(page & (getPageSize() - 1)) && page < HOSTED_PHYSICAL_MEMORY_SIZE && buffer &&
         pread(m_BackingFile, buffer, getPageSize(), page) == static_cast<ssize_t>(getPageSize());
}
bool HostedPhysicalMemoryManager::copyPhysicalPageFromBuffer(physical_uintptr_t page,
                                                             const void* buffer) {
  return page && !(page & (getPageSize() - 1)) && page < HOSTED_PHYSICAL_MEMORY_SIZE && buffer &&
         pwrite(m_BackingFile, buffer, getPageSize(), page) == static_cast<ssize_t>(getPageSize());
}
 
void HostedPhysicalMemoryManager::freePage(physical_uintptr_t page) {
  RecursingLockGuard<Spinlock> guard(m_Lock);
 
  freePageUnlocked(page);
}
void HostedPhysicalMemoryManager::freePageUnlocked(physical_uintptr_t page) {
  if (!m_Lock.acquired())
    FATAL(
        "HostedPhysicalMemoryManager::freePageUnlocked called without an "
        "acquired lock");
 
  // Check for pinned page.
  PageHashable index(page);
  MetadataTable::LookupResult result = m_PageMetadata.lookup(index);
  if (result.hasValue()) {
    struct page p = result.value();
    if (p.active) {
      if (--p.refcount) {
        // Still references.
        m_PageMetadata.update(index, p);
        return;
      } else {
        // No more references, stop tracking page.
        p.active = false;
        m_PageMetadata.update(index, p);
      }
    }
  }
 
  physical_uintptr_t ptr_bitmap = page / getPageSize();
  size_t idx = ptr_bitmap / 32;
  size_t bit = ptr_bitmap % 32;
  if (!(g_PageBitmap[idx] & (1 << bit))) {
    m_Lock.release();
    FATAL_NOLOCK("PhysicalMemoryManager DOUBLE FREE");
  }
 
  g_PageBitmap[idx] &= ~(1 << bit);
 
  m_PageStack.free(page, getPageSize());
  if (Processor::m_Initialised == 2) {
    Metrics::increment(Metrics::PhysicalPageFree);
  }
}
 
void HostedPhysicalMemoryManager::pin(physical_uintptr_t page) {
  RecursingLockGuard<Spinlock> guard(m_Lock);
 
  PageHashable index(page);
  MetadataTable::LookupResult result = m_PageMetadata.lookup(index);
  if (result.hasValue()) {
    struct page p = result.value();
    ++p.refcount;
    p.active = true;
    m_PageMetadata.update(index, p);
  } else {
    struct page p;
    p.refcount = 1;
    p.active = true;
    m_PageMetadata.insert(index, p);
  }
}
 
#if PEDIGREE_HOSTED_SMOKE_TESTS
size_t PhysicalMemoryManager::pageReferenceCountForTest(physical_uintptr_t page) {
  return HostedPhysicalMemoryManager::instance().pageReferenceCountForTestImpl(page);
}
 
size_t HostedPhysicalMemoryManager::pageReferenceCountForTestImpl(physical_uintptr_t page) {
  RecursingLockGuard<Spinlock> guard(m_Lock);
 
  if (page >= HOSTED_PHYSICAL_MEMORY_SIZE) {
    return 0;
  }
 
  const PageHashable index(page);
  const MetadataTable::LookupResult result = m_PageMetadata.lookup(index);
  if (result.hasValue() && result.value().active) {
    return result.value().refcount;
  }
 
  const physical_uintptr_t bitmapPage = page / getPageSize();
  const size_t bitmapIndex = bitmapPage / 32;
  const size_t bitmapBit = bitmapPage % 32;
  return (g_PageBitmap[bitmapIndex] & (1U << bitmapBit)) ? 1 : 0;
}
#endif
 
bool HostedPhysicalMemoryManager::allocateRegion(MemoryRegion& Region, size_t cPages,
                                                 size_t pageConstraints, size_t Flags,
                                                 physical_uintptr_t start) {
  LockGuard<Spinlock> guard(m_RegionLock);
 
  // Allocate a specific physical memory region (always physically continuous)
  if (start != static_cast<physical_uintptr_t>(-1)) {
    // Page-align the start address.
    start &= ~(getPageSize() - 1);
 
    if ((pageConstraints & continuous) != continuous)
      panic("PhysicalMemoryManager::allocateRegion(): function misused");
 
    // Remove the memory from the range-lists (if desired/possible)
    if ((pageConstraints & nonRamMemory) == nonRamMemory) {
      Region.setNonRamMemory(true);
      if (m_PhysicalRanges.allocateSpecific(start, cPages * getPageSize()) == false) {
        if ((pageConstraints & force) != force)
          return false;
        else
          Region.setForced(true);
      }
    } else {
      // Ensure that free() does not attempt to free the given memory...
      Region.setNonRamMemory(true);
      Region.setForced(true);
    }
 
    // Allocate the virtual address space
    uintptr_t vAddress;
 
    if (m_MemoryRegions.allocate(cPages * PhysicalMemoryManager::getPageSize(), vAddress) ==
        false) {
      WARNING("AllocateRegion: MemoryRegion allocation failed.");
      return false;
    }
 
    // Map the physical memory into the allocated space
    VirtualAddressSpace& virtualAddressSpace = Processor::information().getVirtualAddressSpace();
    for (size_t i = 0; i < cPages; i++)
      if (virtualAddressSpace.map(
              start + i * PhysicalMemoryManager::getPageSize(),
              reinterpret_cast<void*>(vAddress + i * PhysicalMemoryManager::getPageSize()),
              Flags) == false) {
        m_MemoryRegions.free(vAddress, cPages * PhysicalMemoryManager::getPageSize());
        WARNING("AllocateRegion: VirtualAddressSpace::map failed.");
        return false;
      }
 
    // Set the memory-region's members
    Region.m_VirtualAddress = reinterpret_cast<void*>(vAddress);
    Region.m_PhysicalAddress = start;
    Region.m_Size = cPages * PhysicalMemoryManager::getPageSize();
    Region.m_bPageBacked = false;
    Region.setAnonymous(pageConstraints & PhysicalMemoryManager::anonymous);
    // Add to the list of memory-regions
    PhysicalMemoryManager::m_MemoryRegions.pushBack(&Region);
    return true;
  } else {
    // Allocate the virtual address space
    uintptr_t vAddress;
    if (m_MemoryRegions.allocate(cPages * PhysicalMemoryManager::getPageSize(), vAddress) ==
        false) {
      WARNING("AllocateRegion: MemoryRegion allocation failed.");
      return false;
    }
 
    const bool virtualOnlyRegion = pageConstraints & virtualOnly;
    if (!virtualOnlyRegion) {
      VirtualAddressSpace& virtualAddressSpace = Processor::information().getVirtualAddressSpace();
 
      // Map the physical memory into the allocated space.
      for (size_t i = 0; i < cPages; i++) {
        physical_uintptr_t page = allocatePage(pageConstraints & addressConstraints);
        if (virtualAddressSpace.map(page, reinterpret_cast<void*>(vAddress + i * getPageSize()),
                                    Flags) == false) {
          freePage(page);
          for (size_t mapped = 0; mapped < i; ++mapped) {
            void* mappedAddress = reinterpret_cast<void*>(vAddress + mapped * getPageSize());
            physical_uintptr_t mappedPage = 0;
            size_t mappedFlags = 0;
            virtualAddressSpace.getMapping(mappedAddress, mappedPage, mappedFlags);
            virtualAddressSpace.unmap(mappedAddress);
            freePage(mappedPage);
          }
          m_MemoryRegions.free(vAddress, cPages * getPageSize());
          WARNING("AllocateRegion: VirtualAddressSpace::map failed.");
          return false;
        }
      }
    }
 
    // Set the memory-region's members
    Region.m_VirtualAddress = reinterpret_cast<void*>(vAddress);
    Region.m_PhysicalAddress = 0;
    Region.m_Size = cPages * PhysicalMemoryManager::getPageSize();
    Region.m_bPageBacked = true;
    Region.setAnonymous(pageConstraints & PhysicalMemoryManager::anonymous);
 
    // Add to the list of memory-regions
    PhysicalMemoryManager::m_MemoryRegions.pushBack(&Region);
    return true;
  }
  return false;
}
 
void HostedPhysicalMemoryManager::initialise(const BootstrapStruct_t& Info) {
  TRACE("Hosted PMM: init");
 
  NOTICE("memory-map:");
 
  size_t pageSize = getPageSize();
 
  // Free pages into the page stack first.
  m_PageStack.free(0, HOSTED_PHYSICAL_MEMORY_SIZE, true);
  m_PageStack.markBelow4GReady();
  TRACE("Hosted PMM: page stack done");
 
  m_PageMetadata.reserve(HOSTED_PHYSICAL_MEMORY_SIZE / pageSize);
 
  // Initialise the free physical ranges
  m_PhysicalRanges.free(0, 0x100000000ULL);
  m_PhysicalRanges.allocateSpecific(0, HOSTED_PHYSICAL_MEMORY_SIZE);
 
// Print the ranges
#if VERBOSE_MEMORY_MANAGER
  NOTICE("physical memory ranges:");
  for (size_t i = 0; i < m_PhysicalRanges.size(); i++) {
    RangeList<uint64_t>::Range range(0, 0);
    if (m_PhysicalRanges.getRange(i, range)) {
      NOTICE(" " << Hex << range.address << " - " << (range.address + range.length));
    }
  }
#endif
 
  // Initialise the range of virtual space for MemoryRegions
  m_MemoryRegions.free(reinterpret_cast<uintptr_t>(KERNEL_VIRTUAL_MEMORYREGION_ADDRESS),
                       KERNEL_VIRTUAL_MEMORYREGION_SIZE);
}
 
void HostedPhysicalMemoryManager::initialisationDone() {
#if PEDIGREE_HOSTED_SMOKE_TESTS
  const size_t trackedBefore = PhysicalMemoryManager::m_MemoryRegions.count();
  MemoryRegion first("Hosted anonymous-region regression");
  if (!allocateRegion(first, 1, PhysicalMemoryManager::anonymous,
                      VirtualAddressSpace::KernelMode)) {
    FATAL("HOSTED-MEMORY-TEST: FAIL anonymous-region allocation");
  }
 
  void* firstAddress = first.virtualAddress();
  VirtualAddressSpace& addressSpace = VirtualAddressSpace::getKernelAddressSpace();
  if (PhysicalMemoryManager::m_MemoryRegions.count() != trackedBefore + 1 ||
      !addressSpace.isMapped(firstAddress)) {
    FATAL("HOSTED-MEMORY-TEST: FAIL anonymous-region ownership");
  }
 
  first.free();
  if (PhysicalMemoryManager::m_MemoryRegions.count() != trackedBefore ||
      addressSpace.isMapped(firstAddress)) {
    FATAL("HOSTED-MEMORY-TEST: FAIL anonymous-region release");
  }
 
  MemoryRegion second("Hosted anonymous-region reuse regression");
  if (!allocateRegion(second, 1, PhysicalMemoryManager::anonymous,
                      VirtualAddressSpace::KernelMode) ||
      second.virtualAddress() != firstAddress) {
    FATAL("HOSTED-MEMORY-TEST: FAIL anonymous-region reuse");
  }
  second.free();
  if (PhysicalMemoryManager::m_MemoryRegions.count() != trackedBefore ||
      addressSpace.isMapped(firstAddress)) {
    FATAL("HOSTED-MEMORY-TEST: FAIL anonymous-region final release");
  }
 
  NOTICE("HOSTED-MEMORY-TEST: PASS anonymous-region-release");
#endif
  NOTICE("PhysicalMemoryManager: kernel initialisation complete");
}
 
HostedPhysicalMemoryManager::HostedPhysicalMemoryManager()
    : m_PhysicalRanges(),
      m_MemoryRegions(),
      m_Lock(false, true),
      m_RegionLock(false, true),
      m_PageMetadata(),
      m_BackingFile(-1) {
  // Create our backing memory file.
  // This lseek/write creates a sparse file on disk.
  m_BackingFile = open("physical.bin", O_RDWR | O_CREAT, 0644);
  lseek(m_BackingFile, HOSTED_PHYSICAL_MEMORY_SIZE - 1, SEEK_SET);
  write(m_BackingFile, "\0", 1);
  lseek(m_BackingFile, 0, SEEK_SET);
}
 
HostedPhysicalMemoryManager::~HostedPhysicalMemoryManager() {
  return;
 
  PhysicalMemoryManager::m_MemoryRegions.clear();
 
  close(m_BackingFile);
  m_BackingFile = -1;
}
 
void HostedPhysicalMemoryManager::unmapRegion(MemoryRegion* pRegion) {
  LockGuard<Spinlock> guard(m_RegionLock);
 
  for (Vector<MemoryRegion*>::Iterator it = PhysicalMemoryManager::m_MemoryRegions.begin();
       it != PhysicalMemoryManager::m_MemoryRegions.end(); it++) {
    if (*it == pRegion) {
      size_t cPages = pRegion->size() / PhysicalMemoryManager::getPageSize();
      uintptr_t start = reinterpret_cast<uintptr_t>(pRegion->virtualAddress());
      physical_uintptr_t phys = pRegion->physicalAddress();
      VirtualAddressSpace& virtualAddressSpace = VirtualAddressSpace::getKernelAddressSpace();
 
      if (pRegion->m_bPageBacked) {
        // Non-contiguous and lazy virtual-only regions own individual pages,
        // not one physical range beginning at m_PhysicalAddress.
      } else if (pRegion->getNonRamMemory()) {
        if (!pRegion->getForced())
          m_PhysicalRanges.free(phys, pRegion->size());
      }
 
      for (size_t i = 0; i < cPages; i++) {
        void* vAddr = reinterpret_cast<void*>(start + i * PhysicalMemoryManager::getPageSize());
        if (!virtualAddressSpace.isMapped(vAddr))
          continue;  // Can happen with virtualOnly mappings.
        physical_uintptr_t pAddr;
        size_t flags;
        virtualAddressSpace.getMapping(vAddr, pAddr, flags);
 
        virtualAddressSpace.unmap(vAddr);
        if (!pRegion->getNonRamMemory() && pRegion->m_bPageBacked)
          freePage(pAddr);
      }
      m_MemoryRegions.free(start, pRegion->size());
      PhysicalMemoryManager::m_MemoryRegions.erase(it);
      break;
    }
  }
}