Processor.h

/*
 * Copyright (c) 2008-2014, Pedigree Developers
 *
 * Please see the CONTRIB file in the root of the source tree for a full
 * list of contributors.
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */
 
#ifndef KERNEL_PROCESSOR_PROCESSOR_H
#define KERNEL_PROCESSOR_PROCESSOR_H
#include "pedigree/kernel/LatencyAccounting.h"
#include "pedigree/kernel/compiler.h"
#include "pedigree/kernel/process/ExecutionContext.h"
#include "pedigree/kernel/processor/ProcessorInformation.h"  // exported
#include "pedigree/kernel/processor/state_forward.h"
#include "pedigree/kernel/processor/types.h"
#include "pedigree/kernel/utilities/StaticString.h"
 
#include <config.h>
 
template <class T>
class Vector;
 
class VirtualAddressSpace;
class BootstrapStruct_t;
 
namespace DebugFlags {
enum FaultType { InstructionFetch = 0, DataWrite = 1, IOReadWrite = 2, DataReadWrite = 3 };
}
 
enum class DeviceHardIrqOperation {
  Schedule,
  SemaphoreAcquire,
  SemaphoreRelease,
  WaitQueueAccess,
  HeapAllocate,
  HeapFree,
};
 
enum class TlbInvalidationResult {
  Success,
  InvalidContext,
  UnsupportedTopology,
  SerialisationTimedOut,
  SubmissionFailed,
  AcknowledgementTimedOut,
  DrainTimedOut,
};
 
class ProcessorBase;
 
class TlbInvalidationGuard {
  friend class ProcessorBase;
 
 public:
  TlbInvalidationGuard() : m_Active(false), m_Global(false) {}
  ~TlbInvalidationGuard();
 
  MUST_USE_RESULT bool closeAdmissionForTerminalFailure(TlbInvalidationResult result);
 
  void retire();
 
  TlbInvalidationGuard(const TlbInvalidationGuard&) = delete;
  TlbInvalidationGuard& operator=(const TlbInvalidationGuard&) = delete;
 
 private:
  bool m_Active;
  bool m_Global;
};
 
#define DEBUG_BREAKPOINT_0 0x01  
#define DEBUG_BREAKPOINT_1 0x02  
#define DEBUG_BREAKPOINT_2 0x04  
#define DEBUG_BREAKPOINT_3 0x08  
#define DEBUG_REG_ACCESS \
  0x2000  
#define DEBUG_SINGLE_STEP \
  0x4000                          
#define DEBUG_TASK_SWITCH 0x8000  
 
extern void apMain() NORETURN;
 
class EXPORTED_PUBLIC ProcessorBase {
  friend class Multiprocessor;
  friend class TlbInvalidationGuard;
  friend class X86GdtManager;
  friend class X64GdtManager;
  friend class Scheduler;
 
 public:
  static void initialise1(const BootstrapStruct_t& Info) INITIALISATION_ONLY;
  static void initialise2(const BootstrapStruct_t& Info) INITIALISATION_ONLY;
  static void initialisationDone();
  static void deinitialise();
  static size_t isInitialised();
 
  static uintptr_t getBasePointer();
  static uintptr_t getStackPointer();
  static uintptr_t getInstructionPointer();
 
  static void switchAddressSpace(VirtualAddressSpace& AddressSpace);
 
  static bool saveState(SchedulerState& state);
  static void restoreState(SchedulerState& state, volatile uintptr_t* pLock = 0) NORETURN;
  static void restoreState(SyscallState& state, volatile uintptr_t* pLock = 0) NORETURN;
#if X64
  static void contextSwitch(InterruptState* state) NORETURN;
#endif
  static void switchState(bool bInterrupts, SchedulerState& a, SchedulerState& b,
                          volatile uintptr_t* pLock = 0);
  static void switchState(bool bInterrupts, SchedulerState& a, SyscallState& b,
                          volatile uintptr_t* pLock = 0);
  static void saveAndJumpKernel(bool bInterrupts, SchedulerState& s, volatile uintptr_t* pLock,
                                uintptr_t address, uintptr_t stack, uintptr_t p1 = 0,
                                uintptr_t p2 = 0, uintptr_t p3 = 0, uintptr_t p4 = 0);
  static void saveAndJumpUser(bool bInterrupts, SchedulerState& s, volatile uintptr_t* pLock,
                              uintptr_t address, uintptr_t stack, uintptr_t p1 = 0,
                              uintptr_t p2 = 0, uintptr_t p3 = 0, uintptr_t p4 = 0);
  static void jumpKernel(volatile uintptr_t* pLock, uintptr_t address, uintptr_t stack,
                         uintptr_t p1 = 0, uintptr_t p2 = 0, uintptr_t p3 = 0,
                         uintptr_t p4 = 0) NORETURN;
  static void jumpUser(volatile uintptr_t* pLock, uintptr_t address, uintptr_t stack,
                       uintptr_t p1 = 0, uintptr_t p2 = 0, uintptr_t p3 = 0,
                       uintptr_t p4 = 0) NORETURN;
 
  static void breakpoint();
  static void halt();
  static void reset();
 
  static size_t getDebugBreakpointCount();
  static uintptr_t getDebugBreakpoint(size_t nBpNumber, DebugFlags::FaultType& nFaultType,
                                      size_t& nLength, bool& bEnabled);
  static void enableDebugBreakpoint(size_t nBpNumber, uintptr_t nLinearAddress,
                                    DebugFlags::FaultType nFaultType, size_t nLength);
  static void disableDebugBreakpoint(size_t nBpNumber);
  static uintptr_t getDebugStatus();
 
  static void haltUntilInterrupt();
  static void pause();
 
  static void setInterrupts(bool bEnable);
  static bool getInterrupts();
 
#if HOSTED
  static void maskInterruptsForSignalReturn();
  static uintptr_t hostedExecutionThreadId();
  static bool onHostedExecutionThread();
  static void enterHostedSignalFrame();
  static void leaveHostedSignalFrame();
  static bool inHostedSignalFrame();
#if PEDIGREE_HOSTED_SMOKE_TESTS
  static size_t hostedSignalFrameDepthForTest();
#endif
#endif
 
  static bool inDeviceHardIrq();
 
  static ExecutionContext executionContext();
 
  static bool guardDeviceHardIrqOperation(DeviceHardIrqOperation operation);
#if HOSTED && PEDIGREE_HOSTED_SMOKE_TESTS
  static size_t deviceHardIrqDepthForTest();
 
  using DeviceHardIrqOperationHook = bool (*)(DeviceHardIrqOperation operation);
 
  static void setDeviceHardIrqOperationHookForTest(DeviceHardIrqOperationHook hook);
 
  static size_t deviceHardIrqOperationDenialsForTest();
 
  enum class HostedContextSwitchStage {
    SwitchStateReturnedMasked,
    SchedulerBookkeepingComplete,
    SchedulerRestoringInterrupts,
  };
 
  using HostedContextSwitchHook = void (*)(HostedContextSwitchStage stage);
 
  static void setHostedContextSwitchHook(HostedContextSwitchHook hook);
 
  static void notifyHostedContextSwitchStage(HostedContextSwitchStage stage);
 
#endif
  static void setSingleStep(bool bEnable, InterruptState& state);
 
  static void invalidate(void* pAddress);
 
  MUST_USE_RESULT static TlbInvalidationResult beginTlbInvalidation(TlbInvalidationGuard& guard);
 
  MUST_USE_RESULT static TlbInvalidationResult invalidateAll(void* pAddress);
 
  MUST_USE_RESULT static TlbInvalidationResult invalidateAll(void* pAddress,
                                                             TlbInvalidationGuard& guard);
 
#if X86_COMMON && !HOSTED
  MUST_USE_RESULT static TlbInvalidationResult invalidateProcessors(void* pAddress,
                                                                    TlbInvalidationGuard& guard,
                                                                    uint64_t processors);
#endif
 
  static bool tlbInvalidationFailureActive();
 
  static bool tlbInvalidationTerminal();
 
  static void invalidateICache(uintptr_t nAddr);
  static void invalidateDCache(uintptr_t nAddr);
  static void flushDCache(uintptr_t nAddr);
  static void flushDCacheAndInvalidateICache(uintptr_t startAddr, uintptr_t endAddr);
 
  static void identify(HugeStaticString& str);
 
  static ProcessorId id();
 
  static size_t index();
 
  static ProcessorInformation& information();
 
  static ProcessorInformation* informationAt(size_t cpu);
 
  static size_t getCount();
 
  static void setTlsBase(uintptr_t newBase);
 
#if X64 && !HOSTED
  static uintptr_t getUserGsBase();
  static void setUserGsBase(uintptr_t newBase);
#endif
 
  static size_t m_Initialised;
 
 private:
  static NEVER_INLINE __attribute__((cold)) bool rejectDeviceHardIrqOperation(
      DeviceHardIrqOperation operation);
 
#if X64
  // Constant-initialized independently of the BSP information constructor.
  static ProcessorInformation::KernelGsAnchor m_BootstrapKernelGsAnchor;
#endif
 
  MUST_USE_RESULT static bool closeTlbInvalidationAdmissionForTerminalFailure(
      TlbInvalidationGuard& guard, TlbInvalidationResult result);
 
  static void endTlbInvalidation(TlbInvalidationGuard& guard);
 
#if HOSTED
  static void _breakpoint();
  static void _reset() NORETURN;
  static void _haltUntilInterrupt();
 
  static bool m_bInterrupts;
#endif
 
  static Vector<ProcessorInformation*> m_ProcessorInformation;
 
  static ProcessorInformation m_SafeBspProcessorInformation;
 
  static size_t m_nProcessors;
};
 
#if X86_COMMON && !HOSTED && !PEDIGREE_BUILDUTILS && defined(IN_PEDIGREE_KERNEL) && \
    !STANDALONE_MUTEXES
ALWAYS_INLINE inline void ProcessorBase::setInterrupts(bool bEnable) {
  if (bEnable) {
#if PEDIGREE_LATENCY_ACCOUNTING
    if (!getInterrupts()) {
      pedigree_irq_time_state(true);
    }
#endif
    // Consume the STI shadow before the caller's next instruction.
    asm volatile("sti\n\tnop" : : : "memory", "cc");
  } else {
    asm volatile("cli" : : : "memory", "cc");
#if PEDIGREE_LATENCY_ACCOUNTING
    pedigree_irq_time_state(false);
#endif
  }
}
 
ALWAYS_INLINE inline bool ProcessorBase::getInterrupts() {
  size_t flags;
  asm volatile("pushf\n\tpop %0" : "=r"(flags) : : "memory", "cc");
  return (flags & 0x200) != 0;
}
#endif
 
inline TlbInvalidationGuard::~TlbInvalidationGuard() {
  retire();
}
 
inline bool TlbInvalidationGuard::closeAdmissionForTerminalFailure(TlbInvalidationResult result) {
  return ProcessorBase::closeTlbInvalidationAdmissionForTerminalFailure(*this, result);
}
 
inline void TlbInvalidationGuard::retire() {
  if (m_Active) {
    ProcessorBase::endTlbInvalidation(*this);
  }
}
 
#include "pedigree/kernel/processor/arm64/Processor.h"       // IWYU pragma: export
#include "pedigree/kernel/processor/armv7/Processor.h"       // IWYU pragma: export
#include "pedigree/kernel/processor/hosted/Processor.h"      // IWYU pragma: export
#include "pedigree/kernel/processor/x64/Processor.h"         // IWYU pragma: export
#include "pedigree/kernel/processor/x86_common/Processor.h"  // IWYU pragma: export
 
#if X64
typedef X64Processor Processor;
#elif ARM64
typedef Arm64Processor Processor;
#elif ARMV7
typedef Armv7Processor Processor;
#elif HOSTED
typedef HostedProcessor Processor;
#else
#error No Processor type could be defined.
#endif
 
ALWAYS_INLINE inline bool ProcessorBase::inDeviceHardIrq() {
  return information().m_DeviceHardIrqDepth != 0;
}
 
ALWAYS_INLINE inline bool ProcessorBase::guardDeviceHardIrqOperation(
    DeviceHardIrqOperation operation) {
  if (LIKELY(!inDeviceHardIrq())) {
    return true;
  }
  return rejectDeviceHardIrqOperation(operation);
}
 
class EnsureInterrupts {
 public:
  EnsureInterrupts(bool desired);
  virtual ~EnsureInterrupts();
 
 private:
  bool m_bPrevious;
};
 
#endif