Thread.h

/*
 * Copyright (c) 2008-2014, Pedigree Developers
 *
 * Please see the CONTRIB file in the root of the source tree for a full
 * list of contributors.
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */
 
#ifndef THREAD_H
#define THREAD_H
#include "pedigree/kernel/ActivityDiagnostics.h"
#include "pedigree/kernel/Spinlock.h"
#include "pedigree/kernel/Subsystem.h"
#include "pedigree/kernel/compiler.h"
#include "pedigree/kernel/process/AtomicStateCleanup.h"
#include "pedigree/kernel/process/CpuAffinity.h"
#include "pedigree/kernel/process/DeferredScope.h"
#include "pedigree/kernel/process/DeferredThreadReap.h"
#include "pedigree/kernel/process/DeferredTimeAccounting.h"
#include "pedigree/kernel/process/Event.h"
#include "pedigree/kernel/process/ExecutionContext.h"
#include "pedigree/kernel/process/ExecutionPersonality.h"
#include "pedigree/kernel/process/SchedulingAlgorithm.h"
#include "pedigree/kernel/process/WaitQueue.h"
#include "pedigree/kernel/processor/ProcessorInformation.h"
#include "pedigree/kernel/processor/Syscalls.h"
#include "pedigree/kernel/processor/VirtualAddressSpace.h"
#include "pedigree/kernel/processor/state_forward.h"
#include "pedigree/kernel/processor/types.h"
#include "pedigree/kernel/utilities/List.h"
#include "pedigree/kernel/utilities/SharedPointer.h"
#include "pedigree/kernel/utilities/assert.h"
#include "pedigree/kernel/utilities/new"
 
#include <config.h>
 
class ExtensibleBitmap;
class PageFaultHandler;
class Process;
struct FilesystemCredentials;
class SyscallManager;
class TerminationDeferral;
class IrqHandlerRegistry;
class TimerHandlerRegistry;
class HostedInterruptManager;
class RoundRobin;
class RequestQueueCallbackScope;
class SchedulerTimerDispatchCleanup;
class UserReturnFrame;
 
#define THREAD_TLS_SIZE PAGE_SIZE
 
class EXPORTED_PUBLIC Thread {
  friend class PerProcessorScheduler;
  friend class Scheduler;
  friend class PageFaultHandler;
  friend class Process;
  friend class WaitQueue;
  friend class Uninterruptible;
  friend class TerminationDeferral;
  friend class SyscallManager;
  friend class IrqHandlerRegistry;
  friend class TimerHandlerRegistry;
  friend class HostedInterruptManager;
  friend class RoundRobin;
  friend class ExecutionContextGuard;
  friend class RequestQueueCallbackScope;
  friend class SchedulerTimerDispatchCleanup;
 
 public:
  class EXPORTED_PUBLIC StackDiscardScope {
   public:
    StackDiscardScope(DeferredScopeRecord::Cleanup cleanup, void* context);
    ~StackDiscardScope();
 
    void disarm();
 
   private:
    NOT_COPYABLE_OR_ASSIGNABLE(StackDiscardScope);
 
    Thread* m_pThread;
    DeferredScopeRecord m_Record;
  };
 
  class EXPORTED_PUBLIC UserReturnFrameScope {
   public:
    UserReturnFrameScope(Thread& owner, UserReturnFrame& frame);
    ~UserReturnFrameScope();
 
   private:
    NOT_COPYABLE_OR_ASSIGNABLE(UserReturnFrameScope);
    static void restore(void* context);
    Thread* m_Owner;
    size_t m_StateLevel;
    UserReturnFrame* m_Frame;
    UserReturnFrame* m_Previous;
    DeferredScopeRecord m_Record;
  };
 
  UserReturnFrame* currentUserReturnFrame() const;
  bool tryRequireSignalFrames();
  void clearSignalFrameRequirement();
  void setUserReturnSignalParked(bool parked);
 
  enum UserReturnWorkFlag : size_t {
    UserReturnExternalWork = 1,
    UserReturnEventsDeferred = 1 << 1,
    UserReturnSignalFrames = 1 << 3,
    UserReturnDeferredException = 1 << 4,
    UserReturnOriginalSyscall = 1 << 5,
  };
 
  bool canSkipUserReturnWork();
  bool clearUserReturnWorkIfIdle();
  bool userReturnWorkPending() const {
    return __atomic_load_n(&m_UserReturnWorkPending, __ATOMIC_ACQUIRE) != 0 ||
           isTerminationDeferred();
  }
  bool requiresSignalFrames() const {
    return __atomic_load_n(&m_SignalFramesRequired, __ATOMIC_ACQUIRE);
  }
 
  class EXPORTED_PUBLIC TemporarySignalMask {
   public:
    TemporarySignalMask(Thread& thread, uint64_t signalMask);
    ~TemporarySignalMask();
 
    bool finish();
 
   private:
    NOT_COPYABLE_OR_ASSIGNABLE(TemporarySignalMask);
 
    static void discard(void* context);
 
    Thread* m_pThread;
    size_t m_StateLevel;
    DeferredScopeRecord m_Record;
  };
 
  enum Status {
    Created,
    Ready,
    Running,
    Sleeping,
    Zombie,
    AwaitingJoin,
  };
 
  enum DebugState {
    None,
    SemWait,
    CondWait,
    Joining,
    FutexWait,
    EventWait,
    ProcessWait,
    CallbackDrain
  };
 
  typedef int (*ThreadStartFunc)(void*);
 
  Thread(Process* pParent, ThreadStartFunc pStartFunction, void* pParam, void* pStack = 0,
         bool semiUser = false, bool bDontPickCore = false, bool delayedStart = false,
         const ThreadPlacement* placement = nullptr);
 
  Thread(Process* pParent);
 
  Thread(Process* pParent, SyscallState& state, bool delayedStart = false,
         const ThreadPlacement* placement = nullptr);
 
  void snapshotPlacement(ThreadPlacement& placement);
  void snapshotPlacementLocked(ThreadPlacement& placement) const;
  AffinityResult requestAffinity(const CpuAffinityMask& mask, uint64_t& generation);
  AffinityResult waitAffinity(uint64_t generation);
  AffinityResult completeAffinityAtSafePoint(bool* waited = nullptr);
  bool affinityWorkPending() const {
    return __atomic_load_n(&m_AffinityReturnPending, __ATOMIC_ACQUIRE) != 0;
  }
#if PEDIGREE_AFFINITY_TESTS
  using AffinityCommitHook = void (*)(Thread*);
  static void setAffinityCommitHookForTest(Thread* target, AffinityCommitHook hook);
#endif
 
  virtual ~Thread();
 
  void shutdown();
 
  void notifySubsystemExit();
  /* Forces the thread to run on the bootstrap processor. */
  void forceToStartupProcessor();
 
  SchedulerState& state();
 
  SchedulerState* pushState();
 
  void popState(bool clean = true);
 
  void abandonCurrentState(bool clean = false);
 
  void abandonAllStates();
 
  VirtualAddressSpace::Stack* getStateUserStack();
 
  void setStateUserStack(VirtualAddressSpace::Stack* st);
 
  void discardUserStackMetadataForExec();
 
  void adoptInitialUserStackForExec(VirtualAddressSpace::Stack* stack);
 
  size_t getStateLevel() const {
    return __atomic_load_n(&m_nStateLevel, __ATOMIC_ACQUIRE);
  }
 
  void* getSyscallDispatchContext() const {
    return m_SyscallDispatchContext;
  }
  void setSyscallDispatchContext(void* context) {
    m_SyscallDispatchContext = context;
  }
 
  void allocateStackAtLevel(size_t stateLevel);
 
  void setKernelStack();
 
  void pokeState(size_t stateLevel, SchedulerState& state);
 
  Process* getParent() const {
    return m_pParent;
  }
 
  void setClearChildTid(uintptr_t address);
 
  uintptr_t takeClearChildTid() {
    return __atomic_exchange_n(&m_ClearChildTid, static_cast<uintptr_t>(0), __ATOMIC_ACQ_REL);
  }
 
  void setRobustList(uintptr_t address, size_t ownerId);
 
  uintptr_t getRobustList() const {
    return __atomic_load_n(&m_RobustList, __ATOMIC_ACQUIRE);
  }
 
  uintptr_t takeRobustList(size_t& ownerId);
 
  void recordTime(CpuTimeMode mode);
 
  void trackTime(CpuTimeMode mode);
 
  void transitionTime(CpuTimeMode from, CpuTimeMode to, bool interruptsAlreadyDisabled = false);
 
  ALWAYS_INLINE void transitionTimeAtInterruptReturn(CpuTimeMode from, CpuTimeMode to) {
    if constexpr (PEDIGREE_TIME_ACCOUNTING && !PEDIGREE_SAMPLED_TIME_ACCOUNTING) {
      ActivityDiagnostics::TimeAccountingScope accountingScope;
      // The architecture boundary owns the physical IRQ mask. Going
      // through CpuTimeSample here could momentarily undo that mask on hosted,
      // where the logical state intentionally describes the pending sigreturn.
      const auto sample = Time::sampleCpuTime();
      const Time::Timestamp elapsed =
          m_TimeAccounting.elapsedAtInterruptDisabled(from, sample.timestamp, sample.processor);
      m_TimeAccounting.recordAtInterruptDisabled(to, sample.timestamp, sample.processor);
      __atomic_store_n(&m_CurrentTimeAccountingMode, static_cast<size_t>(to), __ATOMIC_RELEASE);
      if (elapsed) {
        publishTimeAccounting(from, elapsed);
      }
    } else {
      (void)from;
      __atomic_store_n(&m_CurrentTimeAccountingMode, static_cast<size_t>(to), __ATOMIC_RELEASE);
    }
  }
 
  CpuTimeMode currentTimeAccountingMode() const;
 
  void accountTimerTick(Time::Timestamp delta, bool kernelMode);
 
#if PEDIGREE_BENCHMARK_SYSCALL_TIMING
  static constexpr size_t NoSyscallTimingSlot = ~static_cast<size_t>(0);
 
  size_t installSyscallTimingSlot(size_t slot) {
    return __atomic_exchange_n(&m_ActiveSyscallTimingSlot, slot, __ATOMIC_ACQ_REL);
  }
 
  void restoreSyscallTimingSlot(size_t slot) {
    __atomic_store_n(&m_ActiveSyscallTimingSlot, slot, __ATOMIC_RELEASE);
  }
#endif
 
  Time::Timestamp getUserTime() const {
    return __atomic_load_n(&m_UserTime, __ATOMIC_ACQUIRE);
  }
  Time::Timestamp getKernelTime() const {
    return __atomic_load_n(&m_KernelTime, __ATOMIC_ACQUIRE);
  }
 
#if HOSTED && PEDIGREE_HOSTED_SMOKE_TESTS
  void publishTimeAccountingForHostedTest(Time::Timestamp user, Time::Timestamp system);
#endif
 
  void setParent(Process* p) {
    m_pParent = p;
  }
 
  ExecutionPersonality& executionPersonality() {
    return m_ExecutionPersonality;
  }
 
  Status getStatus() const {
    return m_Status;
  }
 
  bool start();
 
  bool startDetached();
 
  int getExitCode() {
    return m_ExitCode;
  }
 
  void* getKernelStack();
 
  void* getKernelStackBase(size_t* size) const;
 
  size_t getId() {
    return m_Id;
  }
 
  size_t getTaskId() const {
    return __atomic_load_n(&m_TaskId, __ATOMIC_ACQUIRE);
  }
 
  size_t getErrno() {
    return m_StateLevels[m_nStateLevel].m_Errno;
  }
 
  void setErrno(size_t err) {
    m_StateLevels[m_nStateLevel].m_Errno = err;
  }
 
  enum InterruptionReason {
    NotInterrupted,
    InterruptedByTimeout,
    InterruptedBySignal,
  };
 
  bool wasInterrupted() {
    return getInterruptionReason() != NotInterrupted;
  }
 
  void clearInterruption() {
    m_StateLevels[m_nStateLevel].m_InterruptionReason = NotInterrupted;
  }
 
  InterruptionReason getInterruptionReason() {
    return m_StateLevels[m_nStateLevel].m_InterruptionReason;
  }
 
  void setInterruptionReason(InterruptionReason reason) {
    m_StateLevels[m_nStateLevel].m_InterruptionReason = reason;
  }
 
  void markTimeoutInterruptedWait();
 
  void markSignalInterruptedWait();
 
  enum UnwindType {
    Continue = 0,    
    Exit,            
    TerminateThread  
  };
 
  UnwindType getUnwindState() {
    return __atomic_load_n(&m_UnwindState, __ATOMIC_ACQUIRE);
  }
  void setUnwindState(UnwindType ut);
 
  void deferProcessExit(int code);
 
  void deferSignalExit(int signal);
 
  struct DeferredProcessExit {
    int code;
    Subsystem::ExitCause cause;
  };
 
  DeferredProcessExit takeDeferredProcessExit();
 
  bool deferSubsystemException(size_t type, uintptr_t faultAddress, uintptr_t errorCode);
 
  bool hasDeferredSubsystemException() const {
    return __atomic_load_n(&m_DeferredSubsystemExceptionState, __ATOMIC_ACQUIRE) != 0;
  }
 
  bool takeDeferredSubsystemException(size_t& type, uintptr_t& faultAddress, uintptr_t& errorCode);
 
  bool isTerminationDeferred() const {
    return __atomic_load_n(&m_TerminationDeferralDepth, __ATOMIC_ACQUIRE) != 0;
  }
 
  DebugState getDebugState(uintptr_t& address) {
    for (size_t attempt = 0; attempt < 2; ++attempt) {
      const size_t generation = __atomic_load_n(&m_DebugStateGeneration, __ATOMIC_ACQUIRE);
      if (generation & 1) {
        continue;
      }
 
      const DebugState state = __atomic_load_n(&m_DebugState, __ATOMIC_RELAXED);
      const uintptr_t observedAddress = __atomic_load_n(&m_DebugStateAddress, __ATOMIC_RELAXED);
      if (generation == __atomic_load_n(&m_DebugStateGeneration, __ATOMIC_ACQUIRE)) {
        address = observedAddress;
        return state;
      }
    }
 
    address = 0;
    return None;
  }
  void setDebugState(DebugState state, uintptr_t address) {
    __atomic_add_fetch(&m_DebugStateGeneration, static_cast<size_t>(1), __ATOMIC_ACQ_REL);
    __atomic_store_n(&m_DebugStateAddress, address, __ATOMIC_RELAXED);
    __atomic_store_n(&m_DebugState, state, __ATOMIC_RELAXED);
    __atomic_add_fetch(&m_DebugStateGeneration, static_cast<size_t>(1), __ATOMIC_RELEASE);
  }
 
  struct WaitDebugInfo {
    WaitQueue* queue;
    const void* channelOwner;
    uintptr_t channelValue;
    WaitQueue::WakeReason reason;
    size_t stateLevel;
    bool queued;
  };
 
  bool getWaitDebugInfo(WaitDebugInfo& info);
 
  Spinlock& getLock() {
    return m_Lock;
  }
 
  bool sendEvent(Event* pEvent);
 
  void waitForEvent(WaitQueue::StackDiscardCleanup onStackDiscard = nullptr,
                    void* stackDiscardContext = nullptr);
 
  bool waitForEventOrSignalInterruption(WaitQueue::StackDiscardCleanup onStackDiscard = nullptr,
                                        void* stackDiscardContext = nullptr);
 
#if HOSTED && PEDIGREE_HOSTED_SMOKE_TESTS
  enum StateTransitionWindow {
    StatePushBeforePublish,
    StatePushAfterPublish,
    StatePopAfterPublish,
  };
  enum ExternalLeaseReleasePhase {
    ExternalLeaseFinalReleaseUnlocked,
    ExternalLeaseBeforeWaiterWake,
  };
  using StateTransitionHook = void (*)(StateTransitionWindow window, Thread* thread,
                                       size_t previousLevel, size_t nextLevel);
  using JoinOperationHook = void (*)(Thread* target, Process* parent);
  using ExternalLeaseReleaseHook = void (*)(Thread* target, ExternalLeaseReleasePhase phase);
  using DeferredScopeLockHook = void (*)();
  using SignalWaitPreEnrolmentHook = void (*)(Thread* target);
 
  static void setStateTransitionHook(StateTransitionHook hook);
 
  static void setJoinOperationHook(JoinOperationHook hook);
 
  static void setExternalLeaseReleaseHookForHostedTest(Thread* target,
                                                       ExternalLeaseReleaseHook hook);
 
  static void setSignalWaitPreEnrolmentHookForHostedTest(Thread* target,
                                                         SignalWaitPreEnrolmentHook hook);
 
  bool isReapableForHostedTest();
 
  bool wasStartPublishedForHostedTest();
 
  bool waitUntilReapableForHostedTest();
 
  bool runHostedPrequeuedEventRegression();
 
  bool runHostedEventDeliveryLeaseRegression();
 
  bool runHostedEventShutdownRegression();
 
  bool runHostedStatePublicationRegression();
 
  bool runHostedStateCleanupRegression();
 
  bool runHostedExecStackOwnershipRegression();
 
  void withDeferredScopeLockForTest(DeferredScopeLockHook hook);
#endif
 
  void inhibitEvent(size_t eventNumber, bool bInhibit);
 
  uint64_t getSignalMask();
 
  void setSignalMask(uint64_t mask);
 
  uint64_t getSignalMaskForReturnFrame();
 
  void commitSignalHandlerMask(uint64_t mask);
 
  void restoreDeferredSignalMask(size_t stateLevel);
 
  const SyscallState* getOriginalSyscallState() const {
    return m_OriginalSyscallState;
  }
 
  void setOriginalSyscallState(const SyscallState* state) {
    m_OriginalSyscallState = state;
    if (state) {
      __atomic_fetch_or(&m_UserReturnWorkPending, UserReturnOriginalSyscall, __ATOMIC_RELEASE);
    } else {
      __atomic_fetch_and(&m_UserReturnWorkPending, ~static_cast<size_t>(UserReturnOriginalSyscall),
                         __ATOMIC_RELEASE);
    }
  }
 
  void setCurrentSignalDelivery(size_t signalNumber, size_t continuationEpoch);
 
  bool getCurrentSignalDelivery(size_t& signalNumber, size_t& continuationEpoch);
 
  void prepareSignalStateForExec();
 
  bool hasTemporarySignalWaitInterruption();
 
  bool hasActiveTemporarySignalMask();
 
  bool retainTemporarySignalWaitInterruptionOrClear();
 
  struct AlternateSignalStack {
    AlternateSignalStack() : base(0), size(0), enabled(false), inUse(false) {}
 
    uintptr_t base;
    size_t size;
    bool enabled;
    bool inUse;
  };
 
  AlternateSignalStack& getAlternateSignalStack() {
    return m_AlternateSignalStack;
  }
 
  void cullEvent(Event* pEvent);
 
  void cullEvent(size_t eventNumber);
 
  void cullSignalEvent(size_t signalNumber);
 
  bool transferProcessSignalsTo(Thread& target);
 
  bool replaceSignalEvent(size_t signalNumber, Event* replacement, int processDirected = -1,
                          uint64_t rebindGeneration = 0);
 
  bool hasSignalEvent(size_t signalNumber, int processDirected = -1);
 
  bool acceptingEvents();
  uint64_t pendingSignalMask(bool processOnly = false);
  uint64_t pendingSignalOrder(size_t signal, bool processOnly = false);
  Event::Delivery reservePendingSignal(uint64_t mask, bool processOnly = false,
                                       uint64_t expectedSequence = ~uint64_t(0));
  bool restorePendingSignal(Event::Delivery& delivery);
  void cullSignalSource(const void* source);
  void setSynchronousSignalMask(uint64_t mask);
  uint64_t getSynchronousSignalMask();
 
  bool hasEvents();
 
  bool hasEvent(Event* pEvent);
  bool hasEvent(size_t eventNumber);
 
  void setPriority(size_t p) {
    m_Priority = p;
  }
  size_t getPriority() {
    return m_Priority;
  }
 
  void unexpectedExit();
 
  uintptr_t getTlsBase();
 
  void resetTlsBase();
 
#if HOSTED && PEDIGREE_HOSTED_SMOKE_TESTS
  enum TlsResetPhase {
    TlsResetBeforeClear,
    TlsResetCleared,
    TlsResetRemapped,
  };
  using TlsResetHook = void (*)(Thread* thread, TlsResetPhase phase, uintptr_t base);
 
  static void setTlsResetHookForHostedTest(Thread* target, TlsResetHook hook);
#endif
 
  void setTlsBase(uintptr_t base);
 
#if X64 && !HOSTED
  uintptr_t getUserGsBase() const {
    return m_UserGsBase;
  }
  void setUserGsBase(uintptr_t base);
  void saveUserGsBase();
#endif
 
  inline
#if MULTIPROCESSOR
      ProcessorId
#else
      size_t
#endif
      getCpuId() {
    return m_ProcId;
  }
 
  inline void setCpuId(
#if MULTIPROCESSOR
      ProcessorId
#else
      size_t
#endif
          id) {
    m_ProcId = id;
  }
 
  bool join();
 
  bool joinForCompletion();
 
  bool detach();
 
  bool detached() const {
    return m_bDetached;
  }
 
  static void threadExited() NORETURN;
 
  bool eventsDeferred() const;
 
  ExecutionContext executionContext() const;
 
  class PerProcessorScheduler* getScheduler() const {
    return __atomic_load_n(&m_pScheduler, __ATOMIC_ACQUIRE);
  }
 
  const String& getName() const {
    return m_Name;
  }
 
  void setName(const String& name) {
    m_Name = name;
  }
 
  template <size_t N>
  void setName(const char (&name)[N]) {
    m_Name.assign(name, N);
  }
 
#if HOSTED
  size_t enterHostedSignalHandler();
 
  void leaveHostedSignalHandler(size_t stateLevel);
 
  size_t getHostedSignalDepth() const {
    const size_t level = __atomic_load_n(&m_nStateLevel, __ATOMIC_ACQUIRE);
    return __atomic_load_n(&m_StateLevels[level].m_HostedSignalDepth, __ATOMIC_ACQUIRE);
  }
#endif
 
 protected:
  void setScheduler(class PerProcessorScheduler* pScheduler);
 
  void deferEvents();
  void resumeEvents();
  void deferTermination();
  void resumeTermination();
  void markUserReturnWorkFlag(UserReturnWorkFlag flag) {
    __atomic_fetch_or(&m_UserReturnWorkPending, static_cast<size_t>(flag), __ATOMIC_RELEASE);
  }
  void clearUserReturnWorkFlag(UserReturnWorkFlag flag) {
    __atomic_fetch_and(&m_UserReturnWorkPending, ~static_cast<size_t>(flag), __ATOMIC_RELEASE);
  }
  void markUserReturnWorkPending() {
    markUserReturnWorkFlag(UserReturnExternalWork);
  }
  void registerDeferredScope(DeferredScopeRecord& record, bool termination, bool events);
  void armStateCleanup(DeferredScopeRecord& record, DeferredScopeRecord::Cleanup cleanup,
                       void* context);
  void unregisterDeferredScope(DeferredScopeRecord& record);
  void disarmStateCleanup(DeferredScopeRecord& record);
  void unregisterTerminationDeferral(DeferredScopeRecord& record);
  void moveTerminationDeferral(DeferredScopeRecord& from, DeferredScopeRecord& to);
  void retireDeferredScopes(bool allStateLevels, size_t stateLevel = 0);
  size_t stateCleanupCheckpoint();
  void retireDeferredScopesAfter(size_t checkpoint);
  void retireDeferredScopesMatching(bool allStateLevels, size_t stateLevel,
                                    bool newerThanCheckpoint, size_t checkpoint);
  void armAtomicStateCleanup(AtomicStateCleanupRecord& record,
                             AtomicStateCleanupRecord::Cleanup cleanup, void* context);
  void disarmAtomicStateCleanup(AtomicStateCleanupRecord& record);
 
 private:
  void registerFreshTerminationDeferral(DeferredScopeRecord& record,
                                        DeferredScopeRecord::Cleanup cleanup = nullptr,
                                        void* context = nullptr);
 
  Thread(const Thread&);
  Thread& operator=(const Thread&);
 
  void publishTimeAccounting(CpuTimeMode mode, Time::Timestamp elapsed);
 
  void initialisePlacement(const ThreadPlacement* placement);
  void publishReadyNotification();
 
  void cleanStateLevel(size_t level);
 
  size_t beginTemporarySignalMask(uint64_t signalMask);
  bool finishTemporarySignalMask(size_t stateLevel, bool deferForUserReturn = true);
  bool waitForEventInternal(bool stopOnSignalInterruption,
                            WaitQueue::StackDiscardCleanup onStackDiscard,
                            void* stackDiscardContext);
 
  enum class EventSelection {
    AnyDeliverable,
    WithoutExactUserReturn,
    KernelDeliverable,
    StoppedProcessKernel,
  };
 
  bool eventNeedsUserReturnFrameUnlocked(Event* event) const;
  bool eventIsDeliverableUnlocked(Event* event,
                                  EventSelection selection = EventSelection::AnyDeliverable);
 
  bool hasEventsUnlocked(EventSelection selection = EventSelection::AnyDeliverable);
  bool hasDeliverableEventsUnlocked(EventSelection selection = EventSelection::AnyDeliverable);
 
  void markDeferredUserReturnSignalInterruption();
 
  void wakeForDeliverableEvents();
 
  MUST_USE_RESULT Event::Delivery getNextEvent(
      EventSelection selection = EventSelection::AnyDeliverable);
 
  void setStatus(Status s);
  void setStatusUnlocked(Status s);
 
  bool markReapable();
 
  bool joinInternal(bool completion);
 
  struct JoinDiscardContext {
    Thread* target;
    Process* parent;
    bool claimed;
  };
 
  static void discardJoin(void* context);
 
  bool beginExternalLease();
 
  void endExternalLease();
 
  void closeExternalLeaseAdmission();
 
  void closeExternalLeaseAdmissionAndDrain();
 
  bool interruptWaitUnlocked(WaitQueue::WakeReason reason, PerProcessorScheduler*& readyScheduler);
  bool hasActiveWaitUnlocked() const;
  bool hasActiveWaitAtAnyLevel() const;
  bool activeWaitPendingUnlocked() const;
 
  void unlinkWaitsForStackDiscard();
 
  void markTerminalWaitCancelledBeforeBlockUnlocked(size_t level);
  bool consumeTerminalWaitCancelledBeforeBlockUnlocked();
  void clearTerminalWaitCancelledBeforeBlockUnlocked(size_t level);
 
  struct StateLevel {
    StateLevel();
    ~StateLevel();
 
    StateLevel(const StateLevel& s);
    StateLevel& operator=(const StateLevel& s);
 
    SchedulerState* m_State;
    UserReturnFrame* m_UserReturnFrame = nullptr;
 
    VirtualAddressSpace::Stack* m_pKernelStack;
 
    VirtualAddressSpace::Stack* m_pUserStack;
 
    VirtualAddressSpace::Stack* m_pAuxillaryStack;
 
    SharedPointer<ExtensibleBitmap> m_InhibitMask;
 
    uint64_t m_SignalMask;
 
    uint64_t m_SavedSignalMask;
    bool m_TemporarySignalMaskActive;
    bool m_TemporarySignalWaitInterrupted;
    bool m_DeferredSignalMaskRestore;
 
    size_t m_DispatchedSignalNumber;
    size_t m_DispatchedSignalContinuationEpoch;
 
    bool m_bOwnsAlternateSignalStack;
 
    size_t m_Errno;
    InterruptionReason m_InterruptionReason;
 
    bool m_bDispatchingWaitEvent;
 
    ExecutionContextState m_ExecutionContext;
 
    RequestQueueCallbackScope* m_pRequestQueueCallback;
 
    bool m_bTerminalWaitCancelledBeforeBlock;
 
#if HOSTED
    size_t m_HostedSignalDepth;
#endif
 
    WaitQueue::Waiter m_Waiter;
  };
 
  String m_Name;
 
  size_t m_nStateLevel = 0;
 
  Process* m_pParent = nullptr;
  // VFS uses task-local filesystem IDs even when the POSIX identity is shared.
  uint32_t m_FilesystemUid = 0, m_FilesystemGid = 0;
  bool m_FilesystemIdsValid = false;
  const FilesystemCredentials* m_FilesystemOverride = nullptr;
 
  DeferredThreadReapNode m_DeferredReapNode;
 
  ThreadTimeAccounting m_TimeAccounting;
 
  Time::Timestamp m_UserTime = 0;
  Time::Timestamp m_KernelTime = 0;
 
  size_t m_CurrentTimeAccountingMode = static_cast<size_t>(CpuTimeMode::Kernel);
 
#if PEDIGREE_BENCHMARK_SYSCALL_TIMING
  size_t m_ActiveSyscallTimingSlot = NoSyscallTimingSlot;
#endif
 
  void* m_pAllocatedStack = nullptr;
 
  size_t m_Id = 0;
  size_t m_TaskId = 0;
 
  uintptr_t m_DebugStateAddress = 0;
 
  size_t m_DebugStateGeneration = 0;
 
  class PerProcessorScheduler* m_pScheduler = nullptr;
 
  ThreadPlacement m_Placement;
  CpuAffinityMask m_RequestedAffinity;
  WaitQueue m_AffinityWaiters;
  uint64_t m_AffinityGeneration = 0;
  uint64_t m_AffinityCompleted = 0;
  bool m_AffinityPending = false;
  bool m_AffinityGatePending = false;
  bool m_AffinityWorkQueued = false;
  size_t m_AffinityReturnPending = 0;
  bool m_SignalFramesRequired = false;
  bool m_UserReturnSignalParked = false;
  size_t m_UserReturnWorkPending = 0;
  Thread* m_AffinityNext = nullptr;
  bool m_HasSchedulerContext = false;
  bool m_ReadyPublicationPending = false;
 
  size_t m_Priority = DEFAULT_PRIORITY;
 
  Thread* m_pReadyPrevious = nullptr;
  Thread* m_pReadyNext = nullptr;
  size_t m_ReadyQueuePriority = MAX_PRIORITIES;
  bool m_bReadyQueued = false;
 
  void* m_pTlsBase = nullptr;
#if X64 && !HOSTED
  uintptr_t m_UserGsBase = 0;
#endif
 
#if MULTIPROCESSOR
  ProcessorId
#else
  size_t
#endif
      m_ProcId = 0;
 
  Spinlock m_Lock;
 
  WaitQueue m_JoinWaiters;
 
  Spinlock m_ExternalLeaseLock;
 
  WaitQueue m_ExternalLeaseWaiters;
 
  size_t m_nExternalLeases = 0;
 
  bool m_bExternalLeaseAdmissionClosed = false;
 
  bool m_bExternalLeaseReleaseInProgress = false;
 
  WaitQueue m_EventWaiters;
 
  WaitQueue m_EventSenderDrainWaiters;
 
  List<Event*> m_EventQueue;
 
  size_t m_EventSendersInFlight = 0;
 
  StateLevel m_StateLevels[MAX_NESTED_EVENTS];
 
  AlternateSignalStack m_AlternateSignalStack;
 
  const SyscallState* m_OriginalSyscallState = nullptr;
  void* m_SyscallDispatchContext = nullptr;
  size_t m_ActiveSyscalls[serviceEnd] = {};
 
  volatile Status m_Status = Ready;
  ExecutionPersonality m_ExecutionPersonality;
 
  bool m_ExitToIdle = false;
 
  int m_ExitCode = 0;
 
  DebugState m_DebugState = None;
 
  UnwindType m_UnwindState = Continue;
 
  uint64_t m_DeferredProcessExitRequest = 0;
  uint64_t m_SynchronousSignalMask = 0;
 
  size_t m_DeferredSubsystemExceptionState = 0;
  size_t m_DeferredSubsystemExceptionType = 0;
  uintptr_t m_DeferredSubsystemExceptionFaultAddress = 0;
  uintptr_t m_DeferredSubsystemExceptionErrorCode = 0;
 
  bool m_bTlsBaseOverride = false;
 
  bool m_bShutdown = false;
 
  bool m_bSubsystemExitNotified = false;
 
  uintptr_t m_ClearChildTid = 0;
 
  uintptr_t m_RobustList = 0;
  size_t m_RobustListOwnerId = 0;
 
  bool m_bStartRequested = false;
 
  bool m_bDetached = false;
 
  bool m_bJoinClaimed = false;
 
  bool m_bDetachedRetirementClaimed = false;
 
  bool m_bThreadExitRequested = false;
 
  bool m_bExitStarted = false;
 
  bool m_bReapable = false;
 
  bool m_bProcessExitOwned = false;
 
  bool m_bProcessExitParticipant = false;
 
  size_t m_EventDeferralDepth = 0;
 
#if PEDIGREE_LATENCY_ACCOUNTING
  Time::Timestamp m_EventDeferralStarted = 0;
#endif
 
  size_t m_TerminationDeferralDepth = 0;
 
  DeferredScopeRecord* m_pDeferredScopes[MAX_NESTED_EVENTS] = {};
  size_t m_NextStateCleanupSequence = 0;
 
#if HOSTED && PEDIGREE_HOSTED_SMOKE_TESTS
  Spinlock m_DeferredScopeRegressionLock;
#endif
};
 
#endif