File.cc

/*
 * Copyright (c) 2008-2014, Pedigree Developers
 *
 * Please see the CONTRIB file in the root of the source tree for a full
 * list of contributors.
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */
 
#include "File.h"
#include "pedigree/kernel/LockGuard.h"
#include "pedigree/kernel/Log.h"
#include "pedigree/kernel/Metrics.h"
#include "pedigree/kernel/machine/Disk.h"
#include "pedigree/kernel/process/Scheduler.h"
#include "pedigree/kernel/process/Thread.h"
#include "pedigree/kernel/processor/PhysicalMemoryManager.h"
#include "pedigree/kernel/processor/Processor.h"
#include "pedigree/kernel/processor/ProcessorInformation.h"
#include "pedigree/kernel/processor/VirtualAddressSpace.h"
#include "pedigree/kernel/processor/types.h"
#include "pedigree/kernel/syscallError.h"
#include "pedigree/kernel/utilities/Iterator.h"
#include "pedigree/kernel/utilities/Pair.h"
#include "pedigree/kernel/utilities/Result.h"
#include "pedigree/kernel/utilities/Vector.h"
#include "pedigree/kernel/utilities/assert.h"
#include "pedigree/kernel/utilities/utility.h"
 
#include "Filesystem.h"
#include "MemoryMappedFile.h"
#include "VFS.h"
 
RetainedFile::RetainedFile() = default;
 
RetainedFile::RetainedFile(RetainedFile&& other) noexcept : m_File(other.m_File) {
  other.m_File = nullptr;
}
 
RetainedFile::~RetainedFile() {
  reset();
}
 
RetainedFile& RetainedFile::operator=(RetainedFile&& other) noexcept {
  if (this != &other) {
    reset();
    m_File = other.m_File;
    other.m_File = nullptr;
  }
  return *this;
}
 
void RetainedFile::adopt(File* file) {
  reset();
  m_File = file;
}
 
void RetainedFile::reset() {
  File* file = m_File;
  m_File = nullptr;
  if (file)
    file->releaseVfsReference();
}
 
uintptr_t File::futexIdentity() {
  uintptr_t identity = __atomic_load_n(&m_FutexIdentity, __ATOMIC_ACQUIRE);
  if (!identity) {
    static uintptr_t nextIdentity = 0;
    const uintptr_t allocated = __atomic_add_fetch(&nextIdentity, uintptr_t(1), __ATOMIC_RELAXED);
    // Futex keys tag these tokens. Never let wraparound alias a retired file.
    assert(allocated && allocated <= (~uintptr_t(0) >> 1));
    __atomic_compare_exchange_n(&m_FutexIdentity, &identity, allocated, false, __ATOMIC_ACQ_REL,
                                __ATOMIC_ACQUIRE);
    if (!identity) {
      identity = allocated;
    }
  }
  return identity;
}
 
File::ParentLease::ParentLease()
    : m_Parent(nullptr),
      m_Retained(false)
#if THREADS && !defined(STANDALONE_MUTEXES)
      ,
      m_TerminationDeferral(true)
#endif
{
}
 
File::ParentLease::~ParentLease() {
  if (m_Retained) {
    VFS::instance().untrackFile(m_Parent);
  }
}
 
void File::ParentLease::swap(ParentLease& other) {
  File* parent = m_Parent;
  bool retained = m_Retained;
  m_Parent = other.m_Parent;
  m_Retained = other.m_Retained;
  other.m_Parent = parent;
  other.m_Retained = retained;
}
 
bool File::writeCallback(CacheConstants::CallbackCause cause, uintptr_t loc, uintptr_t page,
                         void* meta) {
  File* pFile = reinterpret_cast<File*>(meta);
 
  switch (cause) {
    case CacheConstants::WriteBack: {
      OperationBarrier::Lease operation;
      if (!pFile->tryAcquireFilesystemOperation(operation)) {
        return false;
      }
      pFile->writeBlocks(loc, page, PhysicalMemoryManager::getPageSize());
    } break;
    case CacheConstants::Eviction:
      // Remove this page from our data cache.
      pFile->setCachedPage(loc / pFile->getBlockSize(), FILE_BAD_BLOCK);
      break;
    default:
      WARNING(
          "File: unknown cache callback -- could indicate potential "
          "future I/O issues.");
      break;
  }
  return true;
}
 
bool File::fillCacheCallback(CacheConstants::CallbackCause cause, uintptr_t loc, uintptr_t page,
                             void* meta) {
  File* pFile = reinterpret_cast<File*>(meta);
  if (cause == CacheConstants::WriteBack) {
    OperationBarrier::Lease operation;
    if (!pFile->tryAcquireFilesystemOperation(operation)) {
      return false;
    }
    pFile->writeBlocks(loc, page, PhysicalMemoryManager::getPageSize());
  } else if (cause == CacheConstants::Eviction) {
    pFile->setCachedPage(loc / PhysicalMemoryManager::getPageSize(), FILE_BAD_BLOCK);
  } else if (cause != CacheConstants::Eviction) {
    WARNING("File: unknown fill-cache callback cause.");
  }
  return true;
}
 
File::CacheState::CacheState() : data(FILE_BAD_BLOCK), indexLock(), fill(), fillLock() {}
 
File::CacheState& File::cacheState() {
  return m_CacheState;
}
 
File::File()
    : m_Name(),
      m_AccessedTime(0),
      m_ModifiedTime(0),
      m_CreationTime(0),
      m_Inode(0),
      m_pFilesystem(0),
      m_Size(0),
      m_pParent(0),
      m_MetadataLock(),
      m_pDetachedParent(0),
      m_bDetachedParentHandled(false),
      m_nWriters(0),
      m_nReaders(0),
      m_Uid(0),
      m_Gid(0),
      m_Permissions(0),
      m_CacheState(),
      m_bDirect(false),
      m_WriteLock(),
      m_Lock(),
      m_MonitorTargets() {}
 
File::File(const String& name, Time::Timestamp accessedTime, Time::Timestamp modifiedTime,
           Time::Timestamp creationTime, uintptr_t inode, Filesystem* pFs, size_t size,
           File* pParent)
    : m_Name(name),
      m_AccessedTime(accessedTime),
      m_ModifiedTime(modifiedTime),
      m_CreationTime(creationTime),
      m_Inode(inode),
      m_pFilesystem(pFs),
      m_Size(size),
      m_pParent(pParent),
      m_MetadataLock(),
      m_pDetachedParent(0),
      m_bDetachedParentHandled(false),
      m_nWriters(0),
      m_nReaders(0),
      m_Uid(0),
      m_Gid(0),
      m_Permissions(0),
      m_CacheState(),
      m_bDirect(false),
      m_WriteLock(),
      m_Lock(),
      m_MonitorTargets() {
  size_t maxBlock = size / getBlockSize();
  if (size % getBlockSize()) {
    ++maxBlock;
  }
 
  // Prefill the block cache with bad pages to avoid continuous resizes.
  setCachedPage(maxBlock, FILE_BAD_BLOCK);
}
 
File::~File() {
  closeFileEvents();
  closeReadiness();
 
  {
    LockGuard<Mutex> guard(m_Lock);
    for (auto target : m_MonitorTargets) {
      delete target;
    }
    m_MonitorTargets.clear();
  }
 
  // Releasing the parent can run its destructor, so no File lock may be held.
  if (m_pDetachedParent) {
    VFS::instance().untrackFile(m_pDetachedParent);
  }
}
 
uint64_t File::read(uint64_t location, uint64_t size, uintptr_t buffer, bool bCanBlock) {
  OperationBarrier::Lease operation;
  if (!tryAcquireFilesystemOperation(operation)) {
    return 0;
  }
  Metrics::increment(Metrics::FileReadCalls);
  if (isBytewise()) {
    // Have to perform bytewise reads
    const uint64_t read = readBytewise(location, size, buffer, bCanBlock);
    Metrics::add(Metrics::FileReadBytes, read);
    return read;
  }
 
  LockGuard<Mutex> guard(dataMutationLock());
 
  const size_t fileSize = getSize();
  if (!size || location >= fileSize) {
    return 0;
  }
 
  const uint64_t remaining = fileSize - location;
  if (size > remaining) {
    size = remaining;
  }
 
  const bool filled = useFillCache();
  const size_t blockSize = filled ? PhysicalMemoryManager::getPageSize() : getBlockSize();
 
  size_t n = 0;
  while (size) {
    if (filled && !m_bDirect) {
      const size_t cached = cacheState().fill.read(location, size, buffer);
      if (cached) {
        location += cached;
        size -= cached;
        n += cached;
        if (buffer) {
          buffer += cached;
        }
        continue;
      }
    }
    if (location >= fileSize) {
      Metrics::add(Metrics::FileReadBytes, n);
      return n;
    }
 
    uintptr_t block = location / blockSize;
    uintptr_t offs = location % blockSize;
    uintptr_t sz = (size + offs > blockSize) ? blockSize - offs : size;
 
    // Handle a possible early EOF.
    if (sz > (fileSize - location))
      sz = fileSize - location;
 
    const size_t readAheadLimit = MaxReadPages * blockSize;
    const size_t readAheadBytes =
        size > blockSize ? (size < readAheadLimit - offs ? size + offs : readAheadLimit) : 0;
    uintptr_t buff = readIntoCache(block, false, readAheadBytes);
    if (buff == FILE_BAD_BLOCK) {
      ERROR("File::read - failed to get page from cache, returning early");
      Metrics::add(Metrics::FileReadBytes, n);
      return n;
    }
 
    if (buffer) {
      ForwardMemoryCopy(reinterpret_cast<void*>(buffer), reinterpret_cast<void*>(buff + offs), sz);
      buffer += sz;
    }
    releaseReadReference(block);
 
    location += sz;
    size -= sz;
    n += sz;
  }
  Metrics::add(Metrics::FileReadBytes, n);
  return n;
}
 
size_t File::readCached(uint64_t location, size_t size, uintptr_t buffer,
                        bool (*prepare)(uintptr_t, size_t)) {
  OperationBarrier::Lease operation;
  if (!tryAcquireFilesystemOperation(operation)) {
    return 0;
  }
  Metrics::increment(Metrics::FileCachedReadCalls);
  if (isBytewise() || !useFillCache() || m_bDirect) {
    return 0;
  }
  LockGuard<Mutex> guard(dataMutationLock());
  const size_t fileSize = getSize();
  if (location >= fileSize) {
    return 0;
  }
  if (size > fileSize - location) {
    size = fileSize - location;
  }
  const size_t read = cacheState().fill.read(location, size, buffer, prepare);
  Metrics::add(Metrics::FileReadBytes, read);
  return read;
}
 
uint64_t File::write(uint64_t location, uint64_t size, uintptr_t buffer, bool bCanBlock) {
  WriteGuard guard = lockWrites();
  return guard.write(location, size, buffer, bCanBlock);
}
 
uint64_t File::writeUnlocked(uint64_t location, uint64_t size, uintptr_t buffer, bool bCanBlock) {
  if (!size || location > (~static_cast<uint64_t>(0) - size)) {
    return 0;
  }
 
  const uint64_t endLocation = location + size;
  if (endLocation > static_cast<uint64_t>(~static_cast<size_t>(0))) {
    return 0;
  }
 
  if (isBytewise()) {
    // Have to perform bytewise reads
    return writeBytewise(location, size, buffer, bCanBlock);
  }
 
  if (cacheState().executableMappings) {
    SYSCALL_ERROR(TextFileBusy);
    return 0;
  }
 
  const size_t filesystemBlockSize = getBlockSize();
  const size_t blockSize =
      useFillCache() ? PhysicalMemoryManager::getPageSize() : filesystemBlockSize;
 
  if (!prepareWrite(location, size)) {
    return 0;
  }
 
  size_t n = 0;
  while (size) {
    uintptr_t block = location / blockSize;
    uintptr_t offs = location % blockSize;
    uintptr_t sz = (size + offs > blockSize) ? blockSize - offs : size;
 
    uintptr_t buff = readIntoCache(block, useFillCache() && !offs && sz == blockSize);
    if (buff == FILE_BAD_BLOCK) {
      ERROR("File::write - failed to get page from cache, returning early");
      return n;
    }
 
    ForwardMemoryCopy(reinterpret_cast<void*>(buff + offs), reinterpret_cast<void*>(buffer), sz);
 
    if (useFillCache()) {
      const uint64_t pageOffset = block * blockSize;
      const size_t firstBlock = offs / filesystemBlockSize;
      const size_t endBlock = (offs + sz + filesystemBlockSize - 1) / filesystemBlockSize;
      const size_t fileBlockOffset = firstBlock * filesystemBlockSize;
      writeBlocks(pageOffset + fileBlockOffset, buff + fileBlockOffset,
                  (endBlock - firstBlock) * filesystemBlockSize);
    } else {
      writeBlock(block * blockSize, buff);
    }
    releaseReadReference(block);
 
    location += sz;
    buffer += sz;
    size -= sz;
    n += sz;
  }
 
  if (location >= getSize()) {
    m_Size = location;
    fileAttributeChanged();
  }
  return n;
}
 
uint64_t File::append(uint64_t size, uintptr_t buffer, uint64_t& location, bool bCanBlock) {
  WriteGuard guard = lockWrites();
  return guard.append(size, buffer, location, bCanBlock);
}
 
File::WriteGuard File::lockWrites() {
  return WriteGuard(*this);
}
 
File::WriteGuard::WriteGuard(File& file) : m_File(file), m_Guard(file.writeSerializationLock()) {}
 
File::WriteGuard::~WriteGuard() {
  if (m_MetadataPending) {
    m_File.publishWriteMetadata();
  }
}
 
void File::publishWriteMetadata() {
  OperationBarrier::Lease operation;
  if (!tryAcquireFilesystemOperation(operation)) {
    return;
  }
  if (!isBytewise() && !isDirectory() && !isSymlink() && isSeekable()) {
    Attributes attributes;
    attributes.modified = attributes.changed = Time::getTime();
    updateAttributes(attributes, ModifyTime | ChangeTime);
  }
  publishEvent(FileEvents::Modify);
}
 
uint64_t File::WriteGuard::write(uint64_t location, uint64_t size, uintptr_t buffer, bool bCanBlock,
                                 bool publishMetadata) {
  OperationBarrier::Lease operation;
  if (!m_File.tryAcquireFilesystemOperation(operation)) {
    return 0;
  }
  Metrics::increment(Metrics::FileWriteCalls);
  LockGuard<Mutex> guard(m_File.dataMutationLock());
  const uint64_t written = m_File.writeUnlocked(location, size, buffer, bCanBlock);
  Metrics::add(Metrics::FileWriteBytes, written);
  if (written) {
    if (publishMetadata) {
      m_File.publishWriteMetadata();
    } else {
      m_MetadataPending = true;
    }
  }
  return written;
}
 
uint64_t File::WriteGuard::append(uint64_t size, uintptr_t buffer, uint64_t& location,
                                  bool bCanBlock, bool publishMetadata) {
  OperationBarrier::Lease operation;
  if (!m_File.tryAcquireFilesystemOperation(operation)) {
    return 0;
  }
  Metrics::increment(Metrics::FileWriteCalls);
  LockGuard<Mutex> guard(m_File.dataMutationLock());
  location = m_File.getSize();
  const uint64_t written = m_File.writeUnlocked(location, size, buffer, bCanBlock);
  Metrics::add(Metrics::FileWriteBytes, written);
  if (written) {
    if (publishMetadata) {
      m_File.publishWriteMetadata();
    } else {
      m_MetadataPending = true;
    }
  }
  return written;
}
 
physical_uintptr_t File::getPhysicalPage(size_t offset) {
  OperationBarrier::Lease operation;
  if (!tryAcquireFilesystemOperation(operation)) {
    return ~0UL;
  }
  LockGuard<Mutex> guard(dataMutationLock());
  if (!allowPhysicalPage())
    return ~0UL;
  if (m_bDirect) {
    WARNING("File in direct mode, cannot get backing page.");
    return ~0UL;
  }
 
  EMIT_IF(VFS_NOMMU) {
    return ~0UL;
  }
 
  // Sanitise input.
  size_t blockSize = getBlockSize();
  size_t nativeBlockSize = PhysicalMemoryManager::getPageSize();
  if (useFillCache()) {
    blockSize = nativeBlockSize;
  }
  offset &= ~(blockSize - 1);
 
  // Quick and easy exit.
  if (offset >= getSize()) {
    return ~0UL;
  }
 
  // Check if we have this page in the cache and acquire the cache reference
  // before translating it. A cache address observed before a successful pin
  // may already be retiring.
  uintptr_t vaddr = FILE_BAD_BLOCK;
  bool pinned = false;
  if (LIKELY(!useFillCache())) {
    vaddr = acquireCachedBlock(offset, false);
    if (!vaddr) {
      return ~0UL;
    }
    pinned = true;
  } else {
    // Using the fill cache, because the filesystem has a block size
    // smaller than our native page size. lookup() itself acquires the
    // reference; taking a second pin here would leak one on every mmap.
    vaddr = cacheState().fill.lookup(offset);
    if (!vaddr) {
      return ~0UL;
    }
    pinned = true;
  }
 
  // Translate only while the exact published page remains pinned.
  VirtualAddressSpace& va = Processor::information().getVirtualAddressSpace();
  if (va.isMapped(reinterpret_cast<void*>(vaddr))) {
    physical_uintptr_t phys = 0;
    size_t flags = 0;
    va.getMapping(reinterpret_cast<void*>(vaddr), phys, flags);
    __atomic_add_fetch(&physicalPageLoans(), 1, __ATOMIC_RELEASE);
    return phys;
  }
 
  if (pinned) {
    if (UNLIKELY(useFillCache()))
      cacheState().fill.release(offset);
    else
      unpinBlock(offset);
  }
  return ~0UL;
}
 
void File::returnPhysicalPage(size_t offset) {
  if (m_bDirect) {
    return;
  }
 
  // Sanitise input.
  size_t blockSize = getBlockSize();
  size_t nativeBlockSize = PhysicalMemoryManager::getPageSize();
  if (useFillCache()) {
    blockSize = nativeBlockSize;
  }
  offset &= ~(blockSize - 1);
 
  // Release the page. Beware - this could cause a cache evict, which will
  // make the next read/write at this offset do real (slow) I/O.
  if (UNLIKELY(useFillCache())) {
    cacheState().fill.release(offset);
  } else {
    unpinBlock(offset);
  }
  assert(__atomic_load_n(&physicalPageLoans(), __ATOMIC_ACQUIRE));
  __atomic_sub_fetch(&physicalPageLoans(), 1, __ATOMIC_RELEASE);
}
 
bool File::syncAndReturnPhysicalPage(size_t offset, bool async) {
  const bool succeeded = sync(offset, async);
  returnPhysicalPage(offset);
  return succeeded;
}
 
bool File::tryBeginMappingRelease() {
  return dataMutationLock().tryAcquire();
}
 
void File::endMappingRelease() {
  dataMutationLock().release();
}
 
bool File::sync() {
  return syncRange(0, 0);
}
 
bool File::syncRange(size_t offset, size_t length) {
  OperationBarrier::Lease operation;
  if (!tryAcquireFilesystemOperation(operation)) {
    return false;
  }
  if (length && length - 1 > ~size_t(0) - offset) {
    SYSCALL_ERROR(InvalidArgument);
    return false;
  }
  LockGuard<Mutex> dataGuard(dataMutationLock());
  const bool filled = useFillCache();
  const size_t blockSize = filled ? PhysicalMemoryManager::getPageSize() : getBlockSize();
  const size_t firstBlock = offset / blockSize;
  const size_t lastBlock = length ? (offset + length - 1) / blockSize : ~size_t(0);
  struct SyncPage {
    size_t block;
    uintptr_t buffer;
  };
 
  size_t snapshotSize = 0;
  {
    LockGuard<Mutex> guard(cacheState().indexLock);
    snapshotSize = cacheState().data.count();
  }
 
  // Reserve outside the index lock. Allocation can trigger cache pressure, whose
  // eviction callback removes entries under this same File lock.
  Vector<SyncPage> pages(snapshotSize);
  {
    LockGuard<Mutex> guard(cacheState().indexLock);
    for (auto it = cacheState().data.begin();
         it != cacheState().data.end() && pages.count() < snapshotSize; ++it) {
      const uintptr_t buffer = it.__getNode()->value;
      const size_t block = it.__getNode()->key.hash();
      if (buffer != FILE_BAD_BLOCK && block >= firstBlock && block <= lastBlock) {
        pages.pushBack({block, buffer});
      }
    }
  }
 
  bool succeeded = true;
  uint64_t batch[Disk::MaxSyncPages];
  size_t batchCount = 0;
  auto flushBatch = [&] {
    if (!batchCount)
      return;
    succeeded = syncPages(batch, batchCount) && succeeded;
    for (size_t i = 0; i < batchCount; ++i) {
      if (filled)
        cacheState().fill.release(batch[i]);
      else
        unpinBlock(batch[i]);
    }
    batchCount = 0;
  };
  for (const SyncPage& page : pages) {
    const uint64_t location = page.block * blockSize;
    // The index holds weak identities. Pin each producer page before checking
    // the snapshot, and retain that pin through the batch's durability result.
    if (filled) {
      const uintptr_t buffer = cacheState().fill.lookup(location);
      if (!buffer)
        continue;
      if (buffer != page.buffer) {
        cacheState().fill.release(location);
        continue;
      }
      batch[batchCount++] = location;
      if (batchCount == Disk::MaxSyncPages)
        flushBatch();
      continue;
    }
    if (!pinBlock(location)) {
      continue;
    }
 
    if (getCachedPage(page.block) == page.buffer) {
      writeBlock(location, page.buffer);
      batch[batchCount++] = location;
      if (batchCount == Disk::MaxSyncPages)
        flushBatch();
    } else {
      unpinBlock(location);
    }
  }
  flushBatch();
  return succeeded;
}
 
bool File::syncPages(const uint64_t* offsets, size_t count) {
  OperationBarrier::Lease operation;
  if (!tryAcquireFilesystemOperation(operation)) {
    return false;
  }
  if (count > Disk::MaxSyncPages || (count && !offsets))
    return false;
  for (size_t i = 0; i < count; ++i) {
    if (offsets[i] >= getSize())
      return false;
  }
  bool succeeded = true;
  for (size_t i = 0; i < count; ++i)
    succeeded = sync(offsets[i], false) && succeeded;
  return succeeded;
}
 
bool File::sync(size_t offset, bool async) {
  OperationBarrier::Lease operation;
  if (!tryAcquireFilesystemOperation(operation)) {
    return false;
  }
  return true;
}
 
bool File::tryAcquireFilesystemOperation(OperationBarrier::Lease& operation) const {
  if (m_pFilesystem && !m_pFilesystem->tryAcquireOperation(operation)) {
    SYSCALL_ERROR(DeviceDoesNotExist);
    return false;
  }
  return true;
}
 
Time::Timestamp File::getCreationTime() {
  return getAttributes().changed;
}
 
void File::setCreationTime(Time::Timestamp t) {
  OperationBarrier::Lease operation;
  if (!tryAcquireFilesystemOperation(operation)) {
    return;
  }
  Attributes attributes;
  attributes.changed = t;
  updateAttributes(attributes, ChangeTime);
  publishEvent(FileEvents::Attributes);
}
 
Time::Timestamp File::getAccessedTime() {
  return getAttributes().accessed;
}
 
void File::setAccessedTime(Time::Timestamp t) {
  OperationBarrier::Lease operation;
  if (!tryAcquireFilesystemOperation(operation)) {
    return;
  }
  Attributes attributes;
  attributes.accessed = t;
  updateAttributes(attributes, AccessTime);
  publishEvent(FileEvents::Attributes);
}
 
void File::setTimes(Time::Timestamp accessed, Time::Timestamp modified, bool changeAccessed,
                    bool changeModified) {
  OperationBarrier::Lease operation;
  if (!tryAcquireFilesystemOperation(operation)) {
    return;
  }
  // A caller may request a ctime-only update without changing atime or mtime.
  Attributes attributes;
  attributes.accessed = accessed;
  attributes.modified = modified;
  attributes.changed = Time::getTime();
  updateAttributes(attributes, ChangeTime | (changeAccessed ? AccessTime : 0U) |
                                   (changeModified ? ModifyTime : 0U));
  publishEvent(FileEvents::Attributes);
}
 
Time::Timestamp File::getModifiedTime() {
  return getAttributes().modified;
}
 
void File::setModifiedTime(Time::Timestamp t) {
  OperationBarrier::Lease operation;
  if (!tryAcquireFilesystemOperation(operation)) {
    return;
  }
  Attributes attributes;
  attributes.modified = t;
  updateAttributes(attributes, ModifyTime);
  publishEvent(FileEvents::Attributes);
}
 
String File::getName() const {
  LockGuard<Mutex> guard(m_MetadataLock);
  return m_Name;
}
 
void File::getName(String& s) const {
  LockGuard<Mutex> guard(m_MetadataLock);
  s = m_Name;
}
 
size_t File::getSize() {
  return m_Size;
}
 
void File::setSize(size_t sz) {
  m_Size = sz;
}
 
bool File::isSymlink() {
  return false;
}
 
bool File::isDirectory() {
  return false;
}
 
bool File::isPipe() const {
  return false;
}
 
bool File::isFifo() const {
  return false;
}
 
bool File::isSocket() const {
  return false;
}
 
bool File::isSeekable() const {
  return true;
}
 
bool File::supportsRegularFileOperations() {
  return !isDirectory() && !isSymlink() && !isPipe() && !isFifo() && !isSocket() && !isBytewise() &&
         isSeekable() && !isDirectPhysicalMapping();
}
 
uint64_t File::maximumFileSize() const {
  return ~uint64_t(0) >> 1;
}
 
uintptr_t File::getInode() const {
  return m_Inode;
}
 
void File::setInode(uintptr_t inode) {
  m_Inode = inode;
}
 
Filesystem* File::getFilesystem() const {
  return m_pFilesystem;
}
 
void File::setFilesystem(Filesystem* pFs) {
  m_pFilesystem = pFs;
}
 
void File::fileAttributeChanged() {}
 
FileHandleStatus File::subscribeInodeEvents(FileEventMask, const SharedPointer<FileEventObserver>&,
                                            FileEventSubscription& subscription) {
  subscription.reset();
  return FileHandleStatus::Unsupported;
}
 
void File::publishInodeEvent(const FileEvent&) {}
 
void File::finishInodeRetirement() {}
 
void File::publishEvent(FileEventMask mask, const StringView& name, bool targetIsDirectory) {
  // Deletion must close admission even when no observer is currently registered.
  if (!(mask & FileEvents::DeletedSelf) && !anyFileEventObservers()) {
    return;
  }
  uint32_t producer = 0;
#if THREADS && !defined(STANDALONE_MUTEXES)
  Thread* thread = Processor::information().getCurrentThread();
  if (thread && thread->getParent())
    producer = static_cast<uint32_t>(thread->getParent()->getId());
#endif
  const FileEvent event(mask, name, targetIsDirectory, producer);
  constexpr FileEventMask InodeEvents = FileEvents::Modify | FileEvents::Attributes |
                                        FileEvents::Open | FileEvents::CloseWrite |
                                        FileEvents::CloseNoWrite;
  if (!name.length() && (mask & InodeEvents))
    publishInodeEvent(FileEvent(mask & InodeEvents, name, isDirectory(), producer));
  if (mask & FileEvents::DeletedSelf) {
    notifyFinalFileEvent(event);
  } else {
    notifyFileEvent(event);
  }
 
  constexpr FileEventMask ChildEvents = FileEvents::Access | FileEvents::Modify |
                                        FileEvents::Attributes | FileEvents::CloseWrite |
                                        FileEvents::CloseNoWrite | FileEvents::Open;
  if (!name.length() && (mask & ChildEvents)) {
    ParentLease parent;
    String childName;
    if (snapshotNamespace(parent, childName, mask)) {
      parent.get()->notifyFileEvent(FileEvent(mask, childName.view(), isDirectory(), producer));
    }
  }
}
 
void File::increaseRefCount(bool bIsWriter) {
  if (bIsWriter)
    m_nWriters++;
  else
    m_nReaders++;
}
 
void File::decreaseRefCount(bool bIsWriter) {
  if (bIsWriter)
    m_nWriters--;
  else
    m_nReaders--;
}
 
bool File::retainVfsReference() {
  return VFS::instance().retainTrackedFile(this);
}
 
void File::releaseVfsReference() {
  VFS::instance().untrackFile(this);
}
 
bool File::isStableVfsRoot() const {
  return m_pFilesystem && m_pFilesystem->getRoot() == this;
}
 
void File::setPermissions(uint32_t perms) {
  OperationBarrier::Lease operation;
  if (!tryAcquireFilesystemOperation(operation)) {
    return;
  }
  Attributes attributes;
  attributes.permissions = perms;
  updateAttributes(attributes, Permissions);
  publishEvent(FileEvents::Attributes);
}
 
uint32_t File::getPermissions() const {
  return getAttributes().permissions;
}
 
bool File::setOwnership(size_t uid, size_t gid, bool changeUid, bool changeGid) {
  OperationBarrier::Lease operation;
  if (!tryAcquireFilesystemOperation(operation)) {
    return false;
  }
  if (!changeUid && !changeGid)
    return true;
  if (!changeOwnership(uid, gid, changeUid, changeGid))
    return false;
  publishEvent(FileEvents::Attributes);
  return true;
}
 
bool File::changeOwnership(size_t uid, size_t gid, bool changeUid, bool changeGid) {
  Attributes attributes = getAttributes();
  attributes.uid = uid;
  attributes.gid = gid;
  updateAttributes(attributes, (changeUid ? Owner : 0U) | (changeGid ? Group : 0U));
  return true;
}
 
void File::setUid(size_t uid) {
  OperationBarrier::Lease operation;
  if (!tryAcquireFilesystemOperation(operation)) {
    return;
  }
  Attributes attributes;
  attributes.uid = uid;
  updateAttributes(attributes, Owner);
  publishEvent(FileEvents::Attributes);
}
 
size_t File::getUid() const {
  return getAttributes().uid;
}
 
void File::setGid(size_t gid) {
  OperationBarrier::Lease operation;
  if (!tryAcquireFilesystemOperation(operation)) {
    return;
  }
  Attributes attributes;
  attributes.gid = gid;
  updateAttributes(attributes, Group);
  publishEvent(FileEvents::Attributes);
}
 
size_t File::getGid() const {
  return getAttributes().gid;
}
 
File* File::getParent() const {
  return __atomic_load_n(&m_pParent, __ATOMIC_ACQUIRE);
}
 
void File::getNamespace(ParentLease& parent, String& name) const {
  snapshotNamespace(parent, name, FileEvents::None);
}
 
bool File::snapshotNamespace(ParentLease& parent, String& name, FileEventMask interest) const {
  ParentLease replacement;
  bool captured = false;
  {
    LockGuard<Mutex> guard(m_MetadataLock);
    File* current = getParent();
    if (current) {
      replacement.m_Retained = VFS::instance().retainTrackedFile(current);
      if (replacement.m_Retained || (m_pFilesystem && current == m_pFilesystem->getRoot())) {
        replacement.m_Parent = current;
      }
    }
    // The raw parent pointer is only safe to inspect after acquiring its lease.
    if (!interest ||
        (replacement.m_Parent && replacement.m_Parent->hasFileEventObservers(interest))) {
      name = m_Name;
      captured = true;
    }
  }
  parent.swap(replacement);
  return captured;
}
 
void File::moveNamespace(const String& name, File* parent) {
  LockGuard<Mutex> guard(m_MetadataLock);
  m_Name = name;
  __atomic_store_n(&m_pParent, parent, __ATOMIC_RELEASE);
}
 
void File::retainDetachedParent() {
  LockGuard<Mutex> guard(m_MetadataLock);
  if (__atomic_exchange_n(&m_bDetachedParentHandled, true, __ATOMIC_ACQ_REL)) {
    return;
  }
 
  File* parent = getParent();
  if (!parent) {
    return;
  }
 
  File* root = m_pFilesystem ? m_pFilesystem->getRoot() : nullptr;
  if (parent == root) {
    return;
  }
 
  if (VFS::instance().retainTrackedFile(parent)) {
    m_pDetachedParent = parent;
    return;
  }
 
  // A failed retain for any non-root parent means it is being retired.
  __atomic_store_n(&m_pParent, nullptr, __ATOMIC_RELEASE);
}
 
int File::select(bool bWriting, int timeout) {
  return 1;
}
 
ReadyMask File::queryReady(bool reading, bool writing) {
  ReadyMask ready = ReadyNone;
  if (reading && select(false, 0)) {
    ready |= ReadyRead;
  }
  if (writing && select(true, 0)) {
    ready |= ReadyWrite;
  }
  return ready;
}
 
bool File::supportsReadinessNotifications() const {
  return false;
}
 
bool File::supports(const size_t command) const {
  return false;
}
 
int File::command(const size_t command, void* buffer) {
  return 0;
}
 
size_t File::getBlockSize() const {
  return PhysicalMemoryManager::getPageSize();
}
 
void File::enableDirect() {
  m_bDirect = true;
}
 
void File::disableDirect() {
  m_bDirect = false;
}
 
void File::preallocate(size_t expectedSize, bool zero) {}
 
File::Attributes File::getAttributes() const {
  Attributes attributes;
  {
    LockGuard<Mutex> guard(m_MetadataLock);
    attributes.accessed = m_AccessedTime;
    attributes.modified = m_ModifiedTime;
    attributes.changed = m_CreationTime;
    attributes.uid = m_Uid;
    attributes.gid = m_Gid;
    attributes.permissions = m_Permissions;
  }
  attributes.size = const_cast<File*>(this)->getSize();
  attributes.blocks = attributes.size / 512 + (attributes.size % 512 != 0);
  return attributes;
}
 
XattrStatus File::getExtendedAttribute(const StringView&, void*, size_t, size_t& required) {
  required = 0;
  return XattrStatus::Unsupported;
}
 
XattrStatus File::listExtendedAttributes(void*, size_t, size_t& required) {
  required = 0;
  return XattrStatus::Unsupported;
}
 
XattrStatus File::setExtendedAttribute(const StringView&, const void*, size_t, unsigned) {
  return XattrStatus::Unsupported;
}
 
XattrStatus File::removeExtendedAttribute(const StringView&) {
  return XattrStatus::Unsupported;
}
 
void File::updateAttributes(const Attributes& attributes, uint32_t mask) {
  {
    LockGuard<Mutex> guard(m_MetadataLock);
    if (mask & AccessTime)
      m_AccessedTime = attributes.accessed;
    if (mask & ModifyTime)
      m_ModifiedTime = attributes.modified;
    m_CreationTime = (mask & ChangeTime) ? attributes.changed : Time::getTime();
    if (mask & Owner)
      m_Uid = attributes.uid;
    if (mask & Group)
      m_Gid = attributes.gid;
    if (mask & Permissions)
      m_Permissions = attributes.permissions;
  }
  fileAttributeChanged();
}
 
bool File::prepareSharedMapping(size_t, size_t) {
  return true;
}
 
bool File::prepareWrite(uint64_t location, uint64_t size) {
  const uint64_t end = location + size;
  extend(static_cast<size_t>(end), location, size);
  return getSize() >= end;
}
 
void File::truncate() {}
 
Mutex& File::writeSerializationLock() {
  return m_WriteLock;
}
 
Mutex& File::dataMutationLock() {
  return m_DataMutationLock;
}
 
size_t& File::physicalPageLoans() {
  return m_PhysicalPageLoans;
}
 
bool File::clearDataCache() {
  if (!cacheState().fill.empty()) {
    SYSCALL_ERROR(IoError);
    return false;
  }
  LockGuard<Mutex> guard(cacheState().indexLock);
  cacheState().data.clear();
  return true;
}
 
bool File::allowMapping(bool, bool, bool&) {
  return true;
}
 
bool File::acquireMappingUse(bool executable, bool sharedWrite) {
  LockGuard<Mutex> guard(dataMutationLock());
  CacheState& state = cacheState();
  if ((executable && (sharedWrite || state.sharedWriteMappings)) ||
      (sharedWrite && state.executableMappings)) {
    SYSCALL_ERROR(TextFileBusy);
    return false;
  }
  if (executable)
    ++state.executableMappings;
  if (sharedWrite)
    ++state.sharedWriteMappings;
  return true;
}
 
void File::releaseMappingUse(bool executable, bool sharedWrite) {
  LockGuard<Mutex> guard(dataMutationLock());
  CacheState& state = cacheState();
  if (executable) {
    assert(state.executableMappings);
    --state.executableMappings;
  }
  if (sharedWrite) {
    assert(state.sharedWriteMappings);
    --state.sharedWriteMappings;
  }
}
 
bool File::allowResize(size_t, size_t) {
  return true;
}
 
bool File::resize(size_t size) {
  OperationBarrier::Lease operation;
  if (!tryAcquireFilesystemOperation(operation)) {
    return false;
  }
  if (!supportsRegularFileOperations()) {
    syscallError(isDirectory() ? Error::IsADirectory : Error::InvalidArgument);
    return false;
  }
  LockGuard<Mutex> writeGuard(writeSerializationLock());
#if !VFS_NOMMU
  MemoryMapManager& mappings = MemoryMapManager::instance();
  MemoryMapManager::OperationGuard mappingGuard(mappings);
#endif
  LockGuard<Mutex> guard(dataMutationLock());
  if (m_pFilesystem && m_pFilesystem->isReadOnly()) {
    SYSCALL_ERROR(ReadOnlyFilesystem);
    return false;
  }
  if (cacheState().executableMappings) {
    SYSCALL_ERROR(TextFileBusy);
    return false;
  }
  const size_t oldSize = getSize();
  // Reject backing policy before retiring cache loans or changing mappings.
  if (!allowResize(oldSize, size)) {
    return false;
  }
  if (size == oldSize) {
    return resizeFile(size);
  }
  if (size < oldSize) {
    ShrinkContext context{oldSize, size, nullptr, 0};
    size_t mappingLoans = 0;
#if !VFS_NOMMU
    UniquePointer<MemoryMapManager::PreparedFileResize> mappingPlan;
    const auto mappingStatus = mappings.prepareFileResize(this, oldSize, size, mappingPlan);
    if (mappingStatus != MemoryMapManager::ResizeStatus::Ready) {
      syscallError(mappingStatus == MemoryMapManager::ResizeStatus::NoMemory ? Error::OutOfMemory
                   : mappingStatus == MemoryMapManager::ResizeStatus::Invalid
                       ? Error::InvalidArgument
                       : Error::OperationNotSupported);
      return false;
    }
    context.mappingLoans = mappingPlan.get()->loans();
    context.mappingLoanCount = mappingPlan.get()->loanCount();
    mappingLoans = mappingPlan.get()->totalLoans();
#endif
    // Prefix mappings remain valid borrowers; other physical-page users cannot
    // be revoked by the mapping journal.
    if (__atomic_load_n(&physicalPageLoans(), __ATOMIC_ACQUIRE) != mappingLoans) {
      SYSCALL_ERROR(DeviceBusy);
      return false;
    }
    const size_t pageSize = PhysicalMemoryManager::getPageSize();
    const size_t boundaryOffset = size - size % pageSize;
    const size_t cutoff = boundaryOffset + (size % pageSize ? pageSize : 0);
    if (cutoff < size) {
      SYSCALL_ERROR(InvalidArgument);
      return false;
    }
    UniquePointer<Cache::PreparedDiscard> fillPlan;
    struct BoundaryPage {
      Cache& cache;
      size_t offset;
      uintptr_t address;
      ~BoundaryPage() {
        if (address)
          cache.release(offset);
      }
    } boundary{cacheState().fill, boundaryOffset, 0};
    if (useFillCache()) {
      const auto status = cacheState().fill.prepareDiscardFrom(cutoff, context.mappingLoans,
                                                               context.mappingLoanCount, fillPlan);
      if (status != Cache::DiscardStatus::Ready) {
        syscallError(status == Cache::DiscardStatus::NoMemory  ? Error::OutOfMemory
                     : status == Cache::DiscardStatus::Busy    ? Error::DeviceBusy
                     : status == Cache::DiscardStatus::Invalid ? Error::InvalidArgument
                                                               : Error::IoError);
        return false;
      }
      if (size % pageSize)
        boundary.address = cacheState().fill.lookup(boundaryOffset);
    }
    UniquePointer<PreparedShrink> backend;
    if (!prepareShrink(context, backend))
      return false;
#if !VFS_NOMMU
    mappingPlan.get()->commit();
#endif
    backend.get()->commit();
    if (fillPlan)
      fillPlan.get()->commit();
    if (boundary.address) {
      ByteSet(reinterpret_cast<void*>(boundary.address + size % pageSize), 0,
              pageSize - size % pageSize);
      cacheState().fill.markDirty(boundaryOffset);
    }
    // Native-block backends also cache borrowed addresses in this index.
    // Preserve prefix entries while invalidating every detached suffix alias.
    const size_t cacheBlockSize = useFillCache() ? pageSize : getBlockSize();
    const size_t firstDiscard = cutoff / cacheBlockSize;
    LockGuard<Mutex> indexGuard(cacheState().indexLock);
    for (auto it = cacheState().data.begin(); it != cacheState().data.end();) {
      if (it.__getNode()->key.hash() >= firstDiscard)
        it = cacheState().data.erase(it);
      else
        ++it;
    }
  } else if (!resizeFile(size)) {
    return false;
  }
  if (size > oldSize && useFillCache()) {
    const size_t pageSize = PhysicalMemoryManager::getPageSize();
    const size_t offset = oldSize % pageSize;
    if (offset) {
      const size_t pageOffset = oldSize - offset;
      const uintptr_t buffer = cacheState().fill.lookup(pageOffset);
      if (buffer) {
        const size_t amount =
            (size - oldSize < pageSize - offset) ? size - oldSize : pageSize - offset;
        ByteSet(reinterpret_cast<void*>(buffer + offset), 0, amount);
        cacheState().fill.markDirty(pageOffset);
        cacheState().fill.release(pageOffset);
      }
    }
  }
  Attributes attributes;
  attributes.modified = attributes.changed = Time::getTime();
  updateAttributes(attributes, ModifyTime | ChangeTime);
  publishEvent(FileEvents::Modify);
  return true;
}
 
bool File::prepareShrink(const ShrinkContext&, UniquePointer<PreparedShrink>&) {
  SYSCALL_ERROR(OperationNotSupported);
  return false;
}
 
bool File::resizeFile(size_t) {
  SYSCALL_ERROR(OperationNotSupported);
  return false;
}
 
File* File::open() {
  return this;
}
 
File* File::openForDescriptor(RetainedFile&) {
  return open();
}
 
bool File::isBytewise() const {
  return false;
}
 
uint64_t File::readBytewise(uint64_t location, uint64_t size, uintptr_t buffer, bool bCanBlock) {
  if (isBytewise()) {
    FATAL("A bytewise File subclass didn't implement readBytewise");
  }
  return 0;
}
 
uint64_t File::writeBytewise(uint64_t location, uint64_t size, uintptr_t buffer, bool bCanBlock) {
  if (isBytewise()) {
    FATAL("A bytewise File subclass didn't implement writeBytewise");
  }
  return 0;
}
 
uintptr_t File::readBlock(uint64_t location) {
  String fullPath;
  getFullPath(fullPath);
  ERROR("File: base class readBlock() called for " << fullPath);
  // only truly breaks on debug-enabled builds - in release builds this will
  // just cause an error in the caller
  assert(false);
  return 0;
}
 
void File::writeBlock(uint64_t location, uintptr_t addr) {}
 
void File::writeBlocks(uint64_t location, uintptr_t addr, size_t length) {
  const size_t blockSize = getBlockSize();
  if (!blockSize) {
    return;
  }
 
  for (size_t offset = 0; offset < length; offset += blockSize) {
    writeBlock(location + offset, addr + offset);
  }
}
 
void File::extend(size_t newSize) {
  if (m_Size < newSize)
    m_Size = newSize;
}
 
void File::extend(size_t newSize, uint64_t location, uint64_t size) {
  extend(newSize);
}
 
bool File::pinBlock(uint64_t location) {
  return false;
}
 
uintptr_t File::acquireCachedBlock(uint64_t location, bool retryChanged) {
  const size_t block = location / getBlockSize();
  do {
    const uintptr_t address = getCachedPage(block);
    if (!address || address == FILE_BAD_BLOCK || !pinBlock(location))
      return 0;
    // A key can be replaced between the address snapshot and pinBlock().
    // Only return an address which still names the pinned page.
    if (getCachedPage(block) == address)
      return address;
    unpinBlock(location);
  } while (retryChanged);
  return 0;
}
 
void File::unpinBlock(uint64_t location) {}
 
void File::evict(uint64_t location) {
  setCachedPage(location / getBlockSize(), FILE_BAD_BLOCK);
}
 
void File::setPermissionsOnly(uint32_t perms) {
  m_Permissions = perms;
}
 
void File::setUidOnly(size_t uid) {
  m_Uid = uid;
}
 
void File::setGidOnly(size_t gid) {
  m_Gid = gid;
}
 
void File::dataChanged() {
  EMIT_IF(THREADS) {
    bool bAny = false;
    {
      LockGuard<Mutex> guard(m_Lock);
 
      for (List<MonitorTarget*>::Iterator it = m_MonitorTargets.begin();
           it != m_MonitorTargets.end(); it++) {
        MonitorTarget* pMT = *it;
 
        pMT->pThread->sendEvent(pMT->pEvent);
        delete pMT;
 
        bAny = true;
      }
 
      m_MonitorTargets.clear();
    }
 
    // If anything was waiting on a change, wake it up now.
    if (bAny) {
      Scheduler::instance().yield();
    }
  }
 
  // Readiness observers only receive a change hint and re-query the source.
  // Keep callbacks outside the File lock and the legacy Event registry.
  notifyReadiness(ReadyAll);
}
 
void File::monitor(Thread* pThread, Event* pEvent) {
  assert(pThread);
  assert(pEvent);
 
  EMIT_IF(THREADS) {
    LockGuard<Mutex> guard(m_Lock);
    Event::SendLease registration;
    if (pEvent->tryAcquireRegistration(registration)) {
      m_MonitorTargets.pushBack(
          new MonitorTarget(pThread, pEvent, pedigree_std::move(registration)));
    }
  }
}
 
void File::cullMonitorTargets(Thread* pThread) {
  EMIT_IF(THREADS) {
    LockGuard<Mutex> guard(m_Lock);
 
    for (List<MonitorTarget*>::Iterator it = m_MonitorTargets.begin();
         it != m_MonitorTargets.end();) {
      MonitorTarget* pMT = *it;
 
      if (pMT->pThread == pThread) {
        delete pMT;
        it = m_MonitorTargets.erase(it);
      } else {
        ++it;
      }
    }
  }
}
 
void File::cullMonitorTargets(Event* pEvent) {
  EMIT_IF(THREADS) {
    LockGuard<Mutex> guard(m_Lock);
 
    for (List<MonitorTarget*>::Iterator it = m_MonitorTargets.begin();
         it != m_MonitorTargets.end();) {
      MonitorTarget* pMT = *it;
      if (pMT->pEvent == pEvent) {
        delete pMT;
        it = m_MonitorTargets.erase(it);
      } else {
        ++it;
      }
    }
  }
}
 
void File::getFilesystemLabel(HugeStaticString& s) {
  s = m_pFilesystem->getVolumeLabel();
}
 
void File::getFullPath(String& result, bool bWithMount) {
  HugeStaticString str;
  HugeStaticString tmp;
  str.clear();
  tmp.clear();
 
  File* f = this;
  ParentLease current;
  while (f) {
    ParentLease parent;
    String name;
    f->getNamespace(parent, name);
    if (!parent.get()) {
      break;
    }
    tmp = str;
    str = name;
    if (tmp.length()) {
      str += "/";
      str += tmp;
    }
    current.swap(parent);
    f = current.get();
  }
 
  tmp = str;
  str = "/";
  str += tmp;
 
  if (bWithMount && m_pFilesystem) {
    String mountPath;
    if (VFS::instance().getMountPath(m_pFilesystem, mountPath) && mountPath != "/") {
      tmp = str;
      str = mountPath;
      if (tmp != "/") {
        str += tmp;
      }
    }
  } else if (bWithMount && !m_pFilesystem) {
    ERROR("File::getFullPath called without a filesystem!");
  }
 
  result.assign(str, str.length());
}
 
String File::getFullPath(bool bWithMount) {
  String path;
  getFullPath(path, bWithMount);
  return path;
}
 
uintptr_t File::getCachedPage(size_t block, bool locked) {
  LockGuard<Mutex> guard(cacheState().indexLock, locked);
 
  DataCacheKey key(block);
  auto result = cacheState().data.lookup(key);
  if (result.hasValue()) {
    return result.value();
  } else {
    return FILE_BAD_BLOCK;
  }
}
 
void File::setCachedPage(size_t block, uintptr_t value, bool locked) {
  LockGuard<Mutex> guard(cacheState().indexLock, locked);
 
  assert(value);
 
  DataCacheKey key(block);
  if (value == FILE_BAD_BLOCK) {
    cacheState().data.remove(key);
  } else if (!cacheState().data.update(key, value)) {
    cacheState().data.insert(key, value);
  }
}
 
void File::markPageExternallyWritable(size_t offset) {
  if (useFillCache()) {
    const size_t pageSize = PhysicalMemoryManager::getPageSize();
    cacheState().fill.markExternallyWritable(offset - offset % pageSize);
  }
}
 
size_t File::populateRange(size_t offset, size_t length) {
  OperationBarrier::Lease operation;
  if (!tryAcquireFilesystemOperation(operation)) {
    return 0;
  }
  if (!length || isBytewise())
    return 0;
  if (!useFillCache() || m_bDirect) {
    const size_t pageSize = PhysicalMemoryManager::getPageSize();
    const size_t limit = MaxReadPages * pageSize - offset % pageSize;
    return read(offset, length < limit ? length : limit, 0);
  }
  LockGuard<Mutex> guard(dataMutationLock());
  return populateRangeLocked(offset, length);
}
 
size_t File::populateRangeLocked(size_t offset, size_t length) {
  const size_t fileSize = getSize();
  const size_t pageSize = PhysicalMemoryManager::getPageSize();
  if (!length || offset >= fileSize || !pageSize)
    return 0;
  const size_t within = offset % pageSize;
  const size_t first = offset - within;
  const size_t limit = MaxReadPages * pageSize - within;
  if (length > fileSize - offset)
    length = fileSize - offset;
  if (length > limit)
    length = limit;
  LockGuard<Mutex> fillGuard(cacheState().fillLock);
  Cache& cache = cacheState().fill;
  const uintptr_t demand = cache.lookup(first);
  if (demand) {
    setCachedPage(first / pageSize, demand);
    cache.release(first);
    return length < pageSize - within ? length : pageSize - within;
  }
 
  const size_t count = (within + length - 1) / pageSize + 1;
  ReadPage pages[MaxReadPages] = {};
  ReadPage pending[MaxReadPages] = {};
  size_t pendingSlots[MaxReadPages] = {};
  bool inserted[MaxReadPages] = {};
  size_t pendingCount = 0;
  for (size_t i = 0; i < count; ++i) {
    const size_t location = first + i * pageSize;
    pages[i].offset = location;
    pages[i].buffer = cache.lookup(location);
    if (pages[i].buffer) {
      pages[i].complete = true;
      continue;
    }
    bool existed = false;
    const uintptr_t fresh = cache.insert(location, &existed);
    if (!fresh)
      continue;
    // An Editing page's publication reference alone does not protect it from
    // pressure-driven eviction while the rest of this batch is reserved.
    pages[i].buffer = cache.lookup(location);
    if (!pages[i].buffer)
      continue;
    inserted[i] = !existed;
    if (existed) {
      pages[i].complete = true;
      continue;
    }
    pendingSlots[pendingCount] = i;
    pending[pendingCount++] = pages[i];
  }
  if (pendingCount) {
    const bool allRead = readPages(pending, pendingCount);
    (void)allRead;
    for (size_t i = 0; i < pendingCount; ++i)
      pages[pendingSlots[i]].complete = pending[i].complete;
  }
 
  size_t readyPages = 0;
  while (readyPages < count && pages[readyPages].complete)
    ++readyPages;
  for (size_t i = 0; i < count; ++i) {
    if (!pages[i].buffer)
      continue;
    if (pages[i].complete) {
      if (inserted[i])
        cache.markNoLongerEditing(pages[i].offset);
      setCachedPage(pages[i].offset / pageSize, pages[i].buffer);
    }
    cache.release(pages[i].offset);
    if (inserted[i] && !pages[i].complete) {
      const bool discarded = cache.discardEditing(pages[i].offset);
      (void)discarded;
    }
  }
  if (!readyPages)
    return 0;
  const size_t available = readyPages * pageSize - within;
  return available < length ? available : length;
}
 
bool File::readPages(ReadPage* pages, size_t count) {
  if (count > MaxReadPages || (count && !pages))
    return false;
  bool succeeded = true;
  for (size_t i = 0; i < count; ++i) {
    pages[i].complete = readPage(pages[i].offset, pages[i].buffer);
    succeeded = pages[i].complete && succeeded;
  }
  return succeeded;
}
 
bool File::readPage(uint64_t location, uintptr_t destination) {
  const size_t pageSize = PhysicalMemoryManager::getPageSize();
  const size_t blockSize = getBlockSize();
  if (!blockSize || pageSize % blockSize)
    return false;
  ByteSet(reinterpret_cast<void*>(destination), 0, pageSize);
  const size_t size = getSize();
  for (size_t i = 0; i < pageSize && location < size && i < size - location; i += blockSize) {
    const uintptr_t block = readBlock(location + i);
    if (!block || block == FILE_BAD_BLOCK)
      return false;
    const size_t remaining = size - location - i;
    ForwardMemoryCopy(reinterpret_cast<void*>(destination + i), reinterpret_cast<void*>(block),
                      remaining < blockSize ? remaining : blockSize);
    unpinBlock(location + i);
  }
  return true;
}
 
bool File::useFillCache() const {
  EMIT_IF(VFS_NOMMU) {
#if defined(PEDIGREE_BUILDUTILS)
    if (m_bForceFillCache) {
      return getBlockSize() < PhysicalMemoryManager::getPageSize();
    }
#endif
    // No fill cache in NOMMU builds.
    return false;
  }
  else {
    size_t blockSize = getBlockSize();
    size_t nativeBlockSize = PhysicalMemoryManager::getPageSize();
    return blockSize < nativeBlockSize;
  }
}
 
void File::enableFillCacheWriteback() {
  cacheState().fill.setCallback(fillCacheCallback, this);
}
 
void File::shutdownFillCacheWriteback() {
  cacheState().fill.shutdown();
}
 
bool File::syncFillCache(size_t offset, bool async, bool& present) {
  OperationBarrier::Lease operation;
  if (!tryAcquireFilesystemOperation(operation)) {
    present = false;
    return false;
  }
  const size_t pageSize = PhysicalMemoryManager::getPageSize();
  const size_t pageOffset = offset - (offset % pageSize);
  LockGuard<Mutex> guard(cacheState().fillLock);
  present = cacheState().fill.lookup(pageOffset) != 0;
  if (!present) {
    return true;
  }
 
  const bool succeeded = cacheState().fill.sync(pageOffset, async);
  cacheState().fill.release(pageOffset);
  return succeeded;
}
 
uintptr_t File::readIntoCache(uintptr_t block, bool overwriteWholePage, size_t readAheadBytes) {
  size_t blockSize = getBlockSize();
  size_t nativeBlockSize = PhysicalMemoryManager::getPageSize();
  const bool fillCache = useFillCache();
 
  const size_t offset = block * (fillCache ? nativeBlockSize : blockSize);
 
  if (fillCache) {
    LockGuard<Mutex> fillGuard(cacheState().fillLock);
 
    // Resident reads must not enter insertion's memory-pressure eviction path.
    if (!m_bDirect) {
      const uintptr_t cached = cacheState().fill.lookup(offset);
      if (cached) {
        setCachedPage(block, cached);
        return cached;
      }
      if (readAheadBytes > nativeBlockSize) {
        // Population takes fillLock itself; the caller retains dataMutationLock
        // across this handoff and the final acquisition of the demand page.
        cacheState().fillLock.release();
        fillGuard.disown();
        if (!populateRangeLocked(offset, readAheadBytes))
          return FILE_BAD_BLOCK;
        return readIntoCache(block);
      }
    }
 
    // Using Cache::insert() here is atomic compared to if we did a
    // lookup() followed by an insert() - means we don't need to lock the
    // File object to do this.
    bool didExist = false;
    uintptr_t vaddr = cacheState().fill.insert(offset, nativeBlockSize, &didExist);
    if (!vaddr) {
      return FILE_BAD_BLOCK;
    }
 
    // If in direct mode we are required to read() again
    bool existingReference = false;
    if (didExist) {
      vaddr = cacheState().fill.lookup(offset);
      if (!vaddr) {
        return FILE_BAD_BLOCK;
      }
      if (!m_bDirect) {
        setCachedPage(block, vaddr);
        return vaddr;
      }
      existingReference = true;
      // A fresh backend read must not overwrite deferred writes in this page.
      if (!cacheState().fill.sync(offset, false)) {
        cacheState().fill.release(offset);
        return FILE_BAD_BLOCK;
      }
    }
 
    // The write lock excludes readers until the caller replaces the whole page.
    // Initialising it still gives legacy checksum owners a defined clean image.
    if (overwriteWholePage) {
      ByteSet(reinterpret_cast<void*>(vaddr), 0, nativeBlockSize);
    } else if (!readPage(offset, vaddr)) {
      if (existingReference) {
        cacheState().fill.release(offset);
      }
      if (!didExist && !cacheState().fill.discardEditing(offset)) {
        WARNING("File::readIntoCache could not discard a failed fill for offset " << offset);
      }
      return FILE_BAD_BLOCK;
    }
 
    cacheState().fill.markNoLongerEditing(offset, nativeBlockSize);
 
    if (existingReference) {
      return vaddr;
    }
 
    vaddr = cacheState().fill.lookup(offset);
    if (vaddr) {
      setCachedPage(block, vaddr);
    }
    return vaddr ? vaddr : FILE_BAD_BLOCK;
  }
 
  if (!m_bDirect) {
    const uintptr_t cached = acquireCachedBlock(offset, true);
    if (cached)
      return cached;
  }
  const uintptr_t buff = readBlock(offset);
  if (!buff) {
    ERROR("File::readIntoCache - bad read (" << (block * blockSize) << " - block size is "
                                             << blockSize << ")");
    return FILE_BAD_BLOCK;
  }
 
  if (!m_bDirect) {
    setCachedPage(block, buff);
  }
 
  return buff;
}
 
void File::releaseReadReference(uintptr_t block) {
  if (useFillCache()) {
    cacheState().fill.release(block * PhysicalMemoryManager::getPageSize());
  } else {
    unpinBlock(block * getBlockSize());
  }
}