PerProcessorScheduler.h

/*
 * Copyright (c) 2008-2014, Pedigree Developers
 *
 * Please see the CONTRIB file in the root of the source tree for a full
 * list of contributors.
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */
 
#ifndef PERPROCESSORSCHEDULER_H
#define PERPROCESSORSCHEDULER_H
#include "pedigree/kernel/ActivityDiagnostics.h"
#include "pedigree/kernel/Atomic.h"
#include "pedigree/kernel/compiler.h"
#include "pedigree/kernel/machine/SchedulerTimerHandler.h"
#include "pedigree/kernel/process/ConditionVariable.h"
#include "pedigree/kernel/process/DeferredThreadReap.h"
#include "pedigree/kernel/process/DeferredTimeAccounting.h"
#include "pedigree/kernel/process/Mutex.h"
#include "pedigree/kernel/process/OwnedThread.h"
#include "pedigree/kernel/process/SchedulerWorkerWake.h"
#include "pedigree/kernel/process/Thread.h"
#include "pedigree/kernel/process/WaitQueue.h"
#include "pedigree/kernel/processor/UserReturnFrame.h"
#include "pedigree/kernel/processor/state_forward.h"
#include "pedigree/kernel/processor/types.h"
#include "pedigree/kernel/utilities/IntrusiveMpscQueue.h"
#include "pedigree/kernel/utilities/List.h"
 
#include <config.h>
 
class SchedulingAlgorithm;
class Spinlock;
 
class EXPORTED_PUBLIC PerProcessorScheduler : public SchedulerTimerHandler {
 public:
  enum class StackDiscardReason {
    EmergencyProcessKill,
    HostedRegression,
    LegacyAbiCall,
  };
 
  PerProcessorScheduler();
 
  ~PerProcessorScheduler();
 
  void initialise(Thread* pThread);
 
  void checkEventState(uintptr_t userStack);
 
  void eventHandlerReturned() NORETURN;
 
  void addThread(Thread* pThread, Thread::ThreadStartFunc pStartFunction, void* pParam,
                 bool bUsermode, void* pStack);
 
  void addThread(Thread* pThread, SyscallState& state);
 
  void commitCurrentThreadExit(Spinlock* pLock = 0) NORETURN;
 
  void abandonCurrentThreadStack(StackDiscardReason reason, Spinlock* pLock = 0) NORETURN;
 
  static size_t stackDiscardCount();
 
  static size_t stackDiscardCount(StackDiscardReason reason);
 
  void requestCurrentThreadExitToIdle();
 
  void timer(uint64_t delta, InterruptState& state);
 
  void setClockDeadline(uint64_t deadline);
 
  void removeThread(Thread* pThread);
 
  void threadStatusChanged(Thread* pThread);
 
  size_t logicalCpu() const {
    return m_LogicalCpu;
  }
  bool isIdleThread(const Thread* thread) const {
    return thread == __atomic_load_n(&m_pIdleThread, __ATOMIC_ACQUIRE);
  }
  uint64_t nominalQuantumNs() const {
    return m_NominalQuantumNs;
  }
 
  void ringIrqWorkDoorbell();
 
  void registerWorkerWake(SchedulerWorkerWake& worker, WaitQueue& waiters);
 
  void unregisterWorkerWake(SchedulerWorkerWake& worker);
 
  void ringIrqWorkDoorbell(SchedulerWorkerWake& worker, bool promptOwner = true);
 
  void publishDeferredTimeAccounting();
 
  void serviceIrqWorkDoorbell();
 
  void servicePendingScheduling();
 
  enum class ProcessStopGateMode {
    StopOnly,
    DirectUserTransition,
  };
 
  MUST_USE_RESULT bool serviceProcessStopAtUserReturn(
      ProcessStopGateMode mode = ProcessStopGateMode::StopOnly);
 
  MUST_USE_RESULT bool serviceUserReturnWork(
      InterruptState& state, UserReturnFrame::Origin origin = UserReturnFrame::Origin::Interrupt,
      bool diagnosticSample = false);
 
  MUST_USE_RESULT bool serviceUserReturnWork(
      SyscallState& state, UserReturnFrame::Origin origin = UserReturnFrame::Origin::Syscall,
      bool diagnosticSample = false);
 
  bool sampleUserReturnDiagnostics() {
#if PEDIGREE_ACTIVITY_DIAGNOSTICS
    return (++m_UserReturnDiagnosticSequence % ActivityDiagnostics::UserReturnSamplePeriod) == 0;
#else
    return false;
#endif
  }
 
  void commitUserReturnTerminalState();
 
  void requestIdleThreadWakeup();
 
  void setIdle(Thread* pThread);
 
  void idleUntilInterrupt();
 
#if HOSTED && PEDIGREE_HOSTED_SMOKE_TESTS
  bool runHostedNewThreadWorkerRegressions();
 
  static bool currentIrqWorkDoorbellPendingForTest();
  static void serviceCurrentIrqWorkDoorbellForTest();
#endif
 
 private:
  friend class Scheduler;
  friend class Thread;
  friend class ProcessorThreadAllocator;
  friend class WaitQueue;
 
  void killCurrentThread(Spinlock* pLock = 0) NORETURN;
 
  void schedule(Thread::Status nextStatus = Thread::Ready, bool dispatchEvents = true);
  void scheduleWithInterruptState(Thread::Status nextStatus, bool dispatchEvents,
                                  bool previousInterruptState);
 
  void blockCurrent();
 
  void publishReadyFromWait(Thread* pThread);
 
  void finishCurrentThreadExit(Spinlock* pLock, bool transferToIdle) NORETURN;
 
  void serviceDeferredSubsystemException(InterruptState& state, bool diagnosticSample = false);
 
  void checkEventState(uintptr_t userStack, Thread::EventSelection selection);
 
  void checkEventState(uintptr_t userStack, Thread::EventSelection selection,
                       InterruptState* interruptState, SyscallState* syscallState);
 
  PerProcessorScheduler(const PerProcessorScheduler&);
  PerProcessorScheduler& operator=(const PerProcessorScheduler&);
 
  static void deleteThreadThenRestoreState(Thread* pThread, SchedulerState& newState,
                                           volatile uintptr_t* pLock = 0) NORETURN;
 
  static void deleteThread(Thread* pThread);
 
  void startNewThreadWorker(Process* pParent);
  void stopNewThreadWorker();
 
  void startTimeAccountingWorker(Process* pParent);
  void stopTimeAccountingWorker();
  static int timeAccountingWorkerEntry(void* instance);
  int runTimeAccountingWorker();
  void publishDeferredThreadReap(Thread* thread);
  bool drainDeferredThreadReaps();
  bool enqueueAffinity(Thread* thread, bool accepted = false);
  void drainAffinityRequests();
  void prompt(bool readyPublication = false);
  void armLocalQuantumIfNeeded();
  void updateOneShotTimer();
  void programOneShotTimer();
  Thread* selectNext(Thread* current);
  void serviceWorkerWakeups();
 
  SchedulingAlgorithm* m_pSchedulingAlgorithm;
 
  Mutex m_NewThreadDataLock;
  ConditionVariable m_NewThreadDataCondition;
 
  List<void*> m_NewThreadData;
  List<void*> m_DelayedNewThreadData;
  bool m_NewThreadAdmissionOpen;
  bool m_StopNewThreadWorker;
  OwnedThread m_NewThreadWorker;
  DeferredTimeAccountingWorkerState m_TimeAccountingState;
  DeferredThreadReapNode m_DeferredThreadReapStub;
  IntrusiveMpscQueue<DeferredThreadReapNode, &DeferredThreadReapNode::next> m_DeferredThreadReaps;
  Atomic<size_t> m_nDeferredThreadReaps;
  Atomic<size_t> m_DeferredThreadReapPublicationState;
  Atomic<size_t> m_StopTimeAccountingWorker;
  OwnedThread m_TimeAccountingWorker;
  WaitQueue m_TimeAccountingWorkerWaiters;
  SchedulerWorkerWake m_TimeAccountingWorkerWake;
  Atomic<size_t> m_IrqWorkDoorbell;
  Atomic<size_t> m_ReschedulePending;
  Atomic<size_t> m_RemotePromptPending;
  Atomic<uint64_t> m_ClockDeadline;
  Spinlock m_IrqWorkLock;
  SchedulerWorkerWake* m_pWorkerWakeHead = nullptr;
  Spinlock m_AffinityQueueLock;
  Thread* m_AffinityHead = nullptr;
  Thread* m_AffinityTail = nullptr;
  Atomic<size_t> m_AffinityRequests;
  bool m_AffinityAdmissionOpen = false;
  size_t m_LogicalCpu = ~size_t(0);
  size_t m_PhysicalCpu = 0;
  uint64_t m_NominalQuantumNs = 0;
  uint64_t m_QuantumDeadline = 0;
  uint64_t m_NextLoadSampleDeadline = 0;
  bool m_OneShotTimer = false;
 
#if HOSTED && PEDIGREE_HOSTED_SMOKE_TESTS
  Atomic<size_t> m_nDeferredThreadReapCompletions;
#endif
 
  static constexpr size_t DeferredReapPublicationClosed = static_cast<size_t>(1)
                                                          << ((sizeof(size_t) * 8) - 1);
  static constexpr size_t DeferredReapPublicationCountMask = ~DeferredReapPublicationClosed;
 
  static int processorAddThread(void* instance);
 
  Thread* m_pIdleThread;
  bool m_IdleWakeRequested = false;
  size_t m_SchedulerTickCounter = 0;
  size_t m_UserReturnDiagnosticSequence = 0;
};
 
#endif