Rtc.cc

/*
 * Copyright (c) 2008-2014, Pedigree Developers
 *
 * Please see the CONTRIB file in the root of the source tree for a full
 * list of contributors.
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */
 
#include "Rtc.h"
#if ACPI
#include "Acpi.h"
#endif
#include "pedigree/kernel/BootstrapInfo.h"
#include "pedigree/kernel/LockGuard.h"
#include "pedigree/kernel/Log.h"
#include "pedigree/kernel/TargetInfo.h"
#include "pedigree/kernel/compiler.h"
#include "pedigree/kernel/core/SlamAllocator.h"
#include "pedigree/kernel/machine/IrqManager.h"
#include "pedigree/kernel/machine/Machine.h"
#include "pedigree/kernel/machine/SchedulerTimer.h"
#include "pedigree/kernel/machine/Serial.h"
#include "pedigree/kernel/machine/TimerHandler.h"
#include "pedigree/kernel/panic.h"
#include "pedigree/kernel/process/PerProcessorScheduler.h"
#include "pedigree/kernel/process/Process.h"
#include "pedigree/kernel/process/Scheduler.h"
#include "pedigree/kernel/process/TerminationDeferral.h"
#include "pedigree/kernel/process/Thread.h"
#include "pedigree/kernel/processor/Processor.h"
#include "pedigree/kernel/processor/ProcessorInformation.h"
#include "pedigree/kernel/time/Time.h"
#include "pedigree/kernel/utilities/Iterator.h"
#include "pedigree/kernel/utilities/StaticString.h"
#include "pedigree/kernel/utilities/String.h"
#include "pedigree/kernel/utilities/StringView.h"
#include "pedigree/kernel/utilities/Vector.h"
#include "pedigree/kernel/utilities/assert.h"
#include "pedigree/kernel/utilities/utility.h"
 
#include "RtcCalendar.h"
#include "RtcTimeAccounting.h"
 
class Event;
extern BootstrapStruct_t* g_pBootstrapInfo;
 
// RTC frequency to set at startup - tradeoff between precision of timers
// against constant RTC noise.
#ifdef BOCHS
#define INITIAL_RTC_HZ 64
#else
#define INITIAL_RTC_HZ 512
#endif
 
namespace {
constexpr uint8_t RtcPeriodicInterruptEnable = 1U << 6;
constexpr uint8_t RtcAlarmInterruptEnable = 1U << 5;
constexpr uint8_t RtcUpdateEndedInterruptEnable = 1U << 4;
constexpr uint8_t RtcInterruptEnableMask =
    RtcPeriodicInterruptEnable | RtcAlarmInterruptEnable | RtcUpdateEndedInterruptEnable;
constexpr uint8_t RtcUpdateInhibit = 1U << 7;
constexpr uint8_t RtcInterruptRequested = 1U << 7;
constexpr uint8_t RtcPeriodicFlag = 1U << 6;
// MC146818A UIP can remain asserted for up to 2.228 ms. This margin also
// accommodates compatible devices, emulators, and the uncalibrated TSC used
// during initialise1().
constexpr Time::Timestamp RtcUpdateTimeout = 25 * Time::Multiplier::Millisecond;
constexpr size_t RtcUpdateMaximumPolls = 1000000;
constexpr size_t RtcCalibrationMaximumPolls = 100000000;
 
ALWAYS_INLINE inline uint64_t readOrderedTsc() {
  uint32_t edx = 0;
  uint32_t eax = 0;
  asm volatile("lfence\nrdtsc" : "=d"(edx), "=a"(eax) : : "memory");
  return (static_cast<uint64_t>(edx) << 32U) | eax;
}
 
#if ACPI
bool calibrateTscFromPmTimer(uint64_t& tsc0, uint64_t& tsc1, uint64_t& elapsedNanoseconds) {
  uint16_t port = 0;
  uint32_t mask = 0;
  if (!Acpi::instance().getPmTimerPort(port, mask)) {
    return false;
  }
  IoPort timer("ACPI PM timer calibration");
  if (!timer.allocate(port, 4)) {
    WARNING("RTC: ACPI PM timer port unavailable; using RTC calibration");
    return false;
  }
 
  constexpr uint64_t Frequency = 3579545;
  constexpr uint64_t SampleTicks = Frequency / 10;
  size_t polls = 0;
  for (size_t attempt = 0; attempt < 3; ++attempt) {
    const uint64_t startBefore = readOrderedTsc();
    uint32_t previous = timer.read32() & mask;
    const uint64_t startAfter = readOrderedTsc();
    uint64_t elapsedTicks = 0;
    while (elapsedTicks < SampleTicks) {
      const uint32_t current = timer.read32() & mask;
      // Both counter widths may wrap between reads.
      elapsedTicks += (current - previous) & mask;
      previous = current;
      if (++polls >= RtcCalibrationMaximumPolls) {
        WARNING("RTC: ACPI PM timer calibration timed out; using RTC calibration");
        return false;
      }
      Processor::pause();
    }
    const uint64_t endBefore = readOrderedTsc();
    const uint32_t current = timer.read32() & mask;
    const uint64_t endAfter = readOrderedTsc();
    elapsedTicks += (current - previous) & mask;
 
    if (startAfter < startBefore || endBefore <= startAfter || endAfter < endBefore) {
      continue;
    }
    const uint64_t startSpan = startAfter - startBefore;
    const uint64_t endSpan = endAfter - endBefore;
    tsc0 = startBefore + startSpan / 2;
    tsc1 = endBefore + endSpan / 2;
    // Bound endpoint uncertainty, including host deschedules, to 0.1%.
    if (startSpan + endSpan > (tsc1 - tsc0) / 1000) {
      continue;
    }
    elapsedNanoseconds = elapsedTicks * Time::Multiplier::Second / Frequency;
    return true;
  }
  WARNING("RTC: ACPI PM timer calibration endpoints were delayed; using RTC calibration");
  return false;
}
#endif
}  // namespace
 
Rtc::periodicIrqInfo_t Rtc::periodicIrqInfo[12] = {
    {4, 0x0e, {250000000ULL, 250000000ULL}}, {8, 0x0d, {125000000ULL, 125000000ULL}},
    {16, 0x0c, {62500000ULL, 62500000ULL}},  {32, 0x0b, {31250000ULL, 31250000ULL}},
    {64, 0x0a, {15625000ULL, 15625000ULL}},  {128, 0x09, {7812500ULL, 7812500ULL}},
    {256, 0x08, {3906250ULL, 3906250ULL}},   {512, 0x07, {1953125ULL, 1953125ULL}},
    {1024, 0x06, {976562ULL, 976563ULL}},    {2048, 0x05, {488281ULL, 488281ULL}},
    {4096, 0x04, {244140ULL, 244141ULL}},    {8192, 0x03, {122070ULL, 122070ULL}},
};
 
static uint64_t addAlarmDuration(uint64_t deadline, size_t count, uint64_t multiplier) {
  if (count > ((Time::Infinity - deadline) / multiplier)) {
    return Time::Infinity;
  }
  return deadline + (count * multiplier);
}
 
Rtc Rtc::m_Instance;
 
bool Rtc::CmosTransactionGuard::waitableContext() {
  return Processor::information().getCurrentThread() && Processor::getInterrupts() &&
         !Processor::inDeviceHardIrq();
}
 
Rtc::CmosTransactionGuard::CmosTransactionGuard(Mutex& lock)
    : m_Waitable(waitableContext()),
      m_TerminationDeferral(m_Waitable),
      m_Lock(lock),
      m_Owned(false) {
  if (Processor::inDeviceHardIrq()) {
    panic("RTC CMOS transaction attempted from a hard IRQ");
  }
 
  m_Owned = m_Waitable ? m_Lock.acquireForCompletion() : m_Lock.tryAcquire();
  if (!m_Owned) {
    panic("RTC CMOS transaction contended in an atomic context");
  }
}
 
Rtc::CmosTransactionGuard::~CmosTransactionGuard() {
  if (m_Owned) {
    m_Lock.release();
  }
}
 
static void* currentAlarmDispatchOwner() {
  return RtcAlarmQueue::dispatchOwnerForContext(Processor::information());
}
 
void Rtc::addAlarm(Event* pEvent, size_t alarmSecs, size_t alarmUsecs) {
  Alarm* pAlarm = nullptr;
  m_Lock.acquire();
  pAlarm = m_AlarmQueue.takeReusable();
  m_Lock.release();
 
  if (!pAlarm) {
    pAlarm = new Alarm;
  }
 
  const uint64_t now = getTickCountNano();
  LockGuard<Spinlock> guard(m_Lock);
 
  // Figure out when to trigger the alarm.
  uint64_t target = addAlarmDuration(now, alarmSecs, Time::Multiplier::Second);
  target = addAlarmDuration(target, alarmUsecs, Time::Multiplier::Microsecond);
  pAlarm->prepare(pEvent, target, Processor::information().getCurrentThread());
  m_AlarmQueue.add(pAlarm);
  if (m_DeadlineMode) {
    publishNextDeadlineLocked();
  }
}
 
void Rtc::drainRemoteAlarmDispatch(Event* pEvent, void* owner) {
  Thread* current = Processor::information().getCurrentThread();
  const bool canYield = current && Processor::getInterrupts();
  TerminationDeferral terminationDeferral(canYield);
  uintptr_t previousDebugAddress = 0;
  Thread::DebugState previousDebugState = Thread::None;
  bool debuggingDrain = false;
 
  while (true) {
    m_Lock.acquire();
    const bool dispatching = m_AlarmQueue.hasRemoteInFlight(pEvent, owner);
    m_Lock.release();
 
    if (!dispatching) {
      break;
    }
 
    if (canYield) {
      if (!debuggingDrain) {
        previousDebugState = current->getDebugState(previousDebugAddress);
        current->setDebugState(Thread::CallbackDrain, reinterpret_cast<uintptr_t>(pEvent));
        debuggingDrain = true;
      }
      Scheduler::instance().yield();
    } else {
      Processor::pause();
    }
  }
 
  if (debuggingDrain) {
    current->setDebugState(previousDebugState, previousDebugAddress);
  }
}
 
void Rtc::removeAlarm(Event* pEvent) {
  void* owner = currentAlarmDispatchOwner();
  bool remoteInFlight = false;
  bool selfDeferred = false;
 
  m_Lock.acquire();
  Alarm* reclaim = m_AlarmQueue.removeAllQueued(pEvent, owner, remoteInFlight, selfDeferred);
  m_AlarmQueue.recycleList(reclaim);
  if (m_DeadlineMode) {
    publishNextDeadlineLocked();
  }
  m_Lock.release();
 
  (void)selfDeferred;
  if (remoteInFlight) {
    drainRemoteAlarmDispatch(pEvent, owner);
  }
}
 
size_t Rtc::removeAlarm(class Event* pEvent, bool bRetZero) {
  void* owner = currentAlarmDispatchOwner();
  const uint64_t currTime = getTickCountNano();
 
  m_Lock.acquire();
  RtcAlarmQueue::Removal removal = m_AlarmQueue.removeFirst(pEvent, owner);
  if (removal.record) {
    m_AlarmQueue.recycleList(removal.record);
  }
  if (m_DeadlineMode) {
    publishNextDeadlineLocked();
  }
  m_Lock.release();
 
  if (removal.disposition == RtcAlarmQueue::RemovalDisposition::RemoteInFlight) {
    drainRemoteAlarmDispatch(pEvent, owner);
    return 0;
  }
  if (removal.disposition != RtcAlarmQueue::RemovalDisposition::Removed || bRetZero ||
      removal.deadline < currTime) {
    return 0;
  }
 
  const uint64_t diff = removal.deadline - currTime;
  size_t ret = diff / Time::Multiplier::Second;
  if (diff % Time::Multiplier::Second) {
    ++ret;
  }
  return ret;
}
 
bool Rtc::registerHandler(TimerHandler* handler) {
  return m_HandlerRegistry.registerHandler(handler);
}
 
bool Rtc::unregisterHandler(TimerHandler* handler) {
  const bool removed = m_HandlerRegistry.unregisterHandler(handler);
  if (removed && m_DeadlineMode) {
    LockGuard<Spinlock> guard(m_Lock);
    publishNextDeadlineLocked();
  }
  return removed;
}
 
bool Rtc::armHandler(TimerHandler* handler, uint64_t absoluteDeadlineNs) {
  if (!m_DeadlineMode) {
    return false;
  }
  LockGuard<Spinlock> guard(m_Lock);
  if (!m_HandlerRegistry.armHandler(handler, absoluteDeadlineNs, getTickCountNano())) {
    return false;
  }
  publishNextDeadlineLocked();
  return true;
}
 
void Rtc::publishNextDeadlineLocked() {
  assert(m_DeadlineScheduler);
  uint64_t next = m_AlarmQueue.nextDeadline();
  const uint64_t handlerDeadline = m_HandlerRegistry.nextDeadline();
  if (handlerDeadline < next) {
    next = handlerDeadline;
  }
  m_DeadlineScheduler->setClockDeadline(next == Time::Infinity ? 0 : next);
}
 
void Rtc::deadlineInterrupt() {
  if (!m_DeadlineMode) {
    return;
  }
  m_DeadlinePending = 1;
  m_DeadlineScheduler->ringIrqWorkDoorbell(m_DeadlineWake, false);
}
 
int Rtc::deadlineWorkerEntry(void* context) {
  return reinterpret_cast<Rtc*>(context)->runDeadlineWorker();
}
 
int Rtc::runDeadlineWorker() {
  TerminationDeferral workerLifetime;
  while (!m_StopDeadlineWorker.value()) {
    if (m_DeadlinePending.compareAndSwap(1, 0)) {
      const uint64_t now = getTickCountNano();
      processElapsedTime(now);
      m_HandlerRegistry.dispatchDue(now);
      {
        LockGuard<Spinlock> guard(m_Lock);
        publishNextDeadlineLocked();
      }
      Scheduler::instance().yield();
      continue;
    }
    auto guard = m_DeadlineWaiters.acquire();
    if (!m_DeadlinePending.value() && !m_StopDeadlineWorker.value()) {
      const WaitQueue::WakeReason reason =
          guard.wait(WaitQueue::Channel(), Thread::CondWait, reinterpret_cast<uintptr_t>(this));
      (void)reason;
    }
  }
  return 0;
}
size_t Rtc::getYear() {
  return m_Year;
}
uint8_t Rtc::getMonth() {
  return m_Month;
}
uint8_t Rtc::getDayOfMonth() {
  return m_DayOfMonth;
}
uint8_t Rtc::getDayOfWeek() {
  static size_t monthnumbers[] = {0, 3, 3, 6, 1, 4, 6, 2, 5, 0, 3, 5};
 
  // Calculate day of week
  uint8_t dayOfWeek = m_DayOfMonth % 7;
  dayOfWeek += monthnumbers[m_Month - 1];
  dayOfWeek += ((m_Year % 100) + ((m_Year % 100) / 4)) % 7;
  dayOfWeek -= ((m_Year / 100) % 4 - 3) * 2;
  if (m_Month < 3)
    dayOfWeek--;
  dayOfWeek %= 7;
 
  return dayOfWeek;
}
uint8_t Rtc::getHour() {
  return m_Hour;
}
uint8_t Rtc::getMinute() {
  return m_Minute;
}
uint8_t Rtc::getSecond() {
  return m_Second;
}
uint64_t Rtc::getNanosecond() {
  return m_Nanosecond;
}
Time::Timestamp Rtc::getUnixTimestamp() {
  // initialise3 anchors realtime before enabling deadline mode. Direct
  // filesystem users must see that running clock even when no timer is due.
  return m_DeadlineMode ? Time::getTimeNanoseconds() / Time::Multiplier::Second
                        : Timer::getUnixTimestamp();
}
uint64_t Rtc::getTickCount() {
  return getTickCountNano() / Time::Multiplier::Millisecond;
}
uint64_t Rtc::getTickCountNano() {
  // Migration between the TSC sample and the per-CPU anchor would combine
  // unrelated clock domains. Keep only that bounded snapshot non-preemptible;
  // conversion and global publication do not depend on the current CPU.
  const bool interruptsWereEnabled = Processor::getInterrupts();
  Processor::setInterrupts(false);
  uint64_t anchorTsc = m_Tsc0;
  uint64_t anchorNanoseconds = 0;
  Processor::information().getTscClockAnchor(anchorTsc, anchorNanoseconds);
  const uint64_t tsc = readOrderedTsc();
  Processor::setInterrupts(interruptsWereEnabled);
 
  uint64_t candidate = PcTscClock::fromAnchor(tsc, anchorTsc, anchorNanoseconds, m_TscCalibration);
  const uint64_t coarseFloor = m_TickCount.value();
  if (candidate < coarseFloor) {
    candidate = coarseFloor;
  }
 
  return m_MonotonicTicks.publish(candidate);
}
 
uint64_t Rtc::getTickCountNanoFast() {
  return Rtc::sampleCpuTime().timestamp;
}
 
Time::CpuTimeSample Rtc::sampleCpuTime() {
  // The RTC cursor advances in its worker, after the measured thread has
  // switched out. Accounting needs a clock which advances on this thread.
  // Callers already mask IRQs and reset their baselines on CPU migration,
  // so the immutable local anchor needs no global monotonic publication.
  uint64_t anchorTsc = m_Tsc0;
  uint64_t anchorNanoseconds = 0;
  const ProcessorInformation& processor = Processor::information();
  processor.getTscClockAnchor(anchorTsc, anchorNanoseconds);
  const uint64_t timestamp =
      PcTscClock::fromAnchor(readOrderedTsc(), anchorTsc, anchorNanoseconds, m_TscCalibration);
  return {timestamp, Processor::index()};
}
 
bool Rtc::initialise1(uint8_t centuryIndex) {
  NOTICE("Rtc::initialise1");
 
  // Allocate the I/O port range"CMOS"
  if (m_IoPort.allocate(0x70, 2) == false)
    return false;
 
  // No IRQ yet.
  m_IrqId = 0;
 
  // Initialise handlers.
  m_HandlerRegistry.reset();
 
  if (!RtcCalendar::validCenturyIndex(centuryIndex)) {
    ERROR("RTC: unsupported FADT century index " << Hex << centuryIndex);
    return false;
  }
  m_CenturyIndex = centuryIndex;
  if (centuryIndex) {
    NOTICE("RTC: FADT century index " << Hex << centuryIndex);
  } else {
    NOTICE("RTC: no century register; interpreting years as 1970-2069");
  }
 
  uint8_t statusB = 0;
  if (!read(0x0B, statusB) ||
      !write(0x0B, statusB & ~(RtcUpdateInhibit | RtcInterruptEnableMask | 1U))) {
    return false;
  }
 
  // Find the initial rtc rate
  uint8_t rateBits = 0x06;
  for (size_t i = 0; i < 12; i++)
    if (periodicIrqInfo[i].Hz == INITIAL_RTC_HZ) {
      m_PeriodicIrqInfoIndex = i;
      rateBits = periodicIrqInfo[i].rateBits;
      break;
    }
 
  // Set the Rate for the periodic IRQ
  // Select the standard 32.768 kHz divider and a running clock, regardless
  // of the divider/reset and interrupt state left by firmware.
  uint8_t statusA = 0;
  uint8_t verifiedB = 0;
  if (!write(0x0A, 0x20 | rateBits) || !read(0x0A, statusA) || !read(0x0B, verifiedB) ||
      (statusA & 0x7f) != (0x20 | rateBits) ||
      verifiedB != (statusB & ~(RtcUpdateInhibit | RtcInterruptEnableMask | 1U))) {
    ERROR("RTC: control register configuration did not stick");
    return false;
  }
  if (!readHardwareClock()) {
    ERROR("RTC: initial hardware clock is invalid or unresponsive");
    return false;
  }
  return true;
}
 
bool Rtc::initialise2() {
  NOTICE("Rtc::initialise2");
 
  uint64_t tsc0 = 0;
  uint64_t tsc1 = 0;
  uint64_t elapsedNanoseconds = 0;
  bool usedPmTimer = false;
#if ACPI
  usedPmTimer = calibrateTscFromPmTimer(tsc0, tsc1, elapsedNanoseconds);
#endif
  if (!usedPmTimer) {
    // IRQ8 is still masked. The RTC fallback cannot distinguish coalesced
    // periods, so prefer the cumulative ACPI counter when it is available.
    constexpr size_t CalibrationPeriods = 50;
    setPeriodicInterruptEnabled(true);
    size_t calibrationPolls = 0;
    while (!(read(0x0C) & RtcPeriodicFlag)) {
      if (++calibrationPolls >= RtcCalibrationMaximumPolls) {
        setPeriodicInterruptEnabled(false);
        ERROR("RTC: timed out waiting for TSC calibration to start");
        return false;
      }
      Processor::pause();
    }
 
    tsc0 = readOrderedTsc();
 
    size_t periods = 0;
    while (periods < CalibrationPeriods) {
      if (read(0x0C) & RtcPeriodicFlag) {
        ++periods;
      }
      if (++calibrationPolls >= RtcCalibrationMaximumPolls) {
        setPeriodicInterruptEnabled(false);
        ERROR("RTC: timed out collecting periodic TSC calibration samples");
        return false;
      }
      Processor::pause();
    }
 
    tsc1 = readOrderedTsc();
    setPeriodicInterruptEnabled(false);
 
    for (size_t i = 0; i < CalibrationPeriods; ++i) {
      elapsedNanoseconds += periodicIrqInfo[m_PeriodicIrqInfoIndex].ns[i & 1];
    }
  }
  if (tsc1 <= tsc0 || !elapsedNanoseconds) {
    ERROR("RTC: invalid TSC calibration interval");
    return false;
  }
  const uint64_t elapsedCycles = tsc1 - tsc0;
  m_TscCalibration = PcTscClock::Calibration(elapsedCycles, elapsedNanoseconds);
  NOTICE("TSC calibration: " << elapsedCycles << " cycles / " << elapsedNanoseconds << " ns ("
                             << (usedPmTimer ? "ACPI PM timer" : "RTC") << ")");
 
  m_TickCount = 0;
  m_ProcessedTickCount = 0;
  m_Tsc0 = tsc1;
  m_MonotonicTicks.reset();
  Processor::information().initialiseTscClockAnchor(tsc1, 0);
#if PEDIGREE_LATENCY_ACCOUNTING
  const bool interruptsWereEnabled = Processor::getInterrupts();
  Processor::setInterrupts(false);
  LatencyAccounting::armCpu();
  Processor::setInterrupts(interruptsWereEnabled);
#endif
 
  return true;
}
 
void Rtc::initialiseProcessorClock() {
  const bool interruptsWereEnabled = Processor::getInterrupts();
  Processor::setInterrupts(false);
  ProcessorInformation& processor = Processor::information();
  const uint64_t tsc = readOrderedTsc();
  uint64_t nanoseconds = m_MonotonicTicks.value();
  const uint64_t coarseFloor = m_TickCount.value();
  if (nanoseconds < coarseFloor) {
    nanoseconds = coarseFloor;
  }
  processor.initialiseTscClockAnchor(tsc, nanoseconds);
  LatencyAccounting::armCpu();
  Processor::setInterrupts(interruptsWereEnabled);
}
 
bool Rtc::initialise3() {
  size_t selectedIndex = m_PeriodicIrqInfoIndex;
  bool rateSpecified = false;
  const char* commandLine = g_pBootstrapInfo->getCommandLine();
  if (commandLine) {
    Vector<String> arguments = String(commandLine).tokenise(' ');
    for (const auto& argument : arguments) {
      const StringView view = argument.view();
      constexpr size_t PrefixLength = 9;
      if (!(view == "--rtc-hz") &&
          (view.length() < PrefixLength || !(view.substring(0, PrefixLength) == "--rtc-hz=")))
        continue;
 
      size_t hz = 0;
      bool valid = !rateSpecified && view.length() > PrefixLength;
      for (size_t i = PrefixLength; valid && i < view.length(); ++i) {
        const char digit = view[i];
        valid = digit >= '0' && digit <= '9' && hz <= 8192;
        if (valid)
          hz = hz * 10 + static_cast<size_t>(digit - '0');
      }
      size_t index = 0;
      while (index < 12 && periodicIrqInfo[index].Hz != hz)
        ++index;
      if (!valid || index == 12) {
        ERROR("RTC: invalid or repeated --rtc-hz option: " << argument);
        return false;
      }
      selectedIndex = index;
      rateSpecified = true;
    }
  }
 
  // Keep boot TSC calibration identical across runtime-rate experiments.
  // Periodic interrupts are still disabled and IRQ8 has no handler yet.
  const uint8_t statusA = 0x20 | periodicIrqInfo[selectedIndex].rateBits;
  uint8_t verifiedA = 0;
  if (!write(0x0A, statusA) || !read(0x0A, verifiedA) || (verifiedA & 0x7f) != statusA) {
    ERROR("RTC: runtime interrupt frequency did not stick");
    return false;
  }
  m_PeriodicIrqInfoIndex = selectedIndex;
  SchedulerTimer* schedulerTimer = Machine::instance().getSchedulerTimer();
  if (schedulerTimer && schedulerTimer->supportsOneShot()) {
    m_DeadlineScheduler = Scheduler::schedulerForCpu(0);
    if (!m_DeadlineScheduler) {
      return false;
    }
    const uint64_t baseline = getTickCountNano();
    m_TickCount = baseline;
    m_ProcessedTickCount = baseline;
    if (!Time::anchorRealtime(Time::getTimeNanoseconds())) {
      return false;
    }
    m_DeadlineScheduler->registerWorkerWake(m_DeadlineWake, m_DeadlineWaiters);
    Thread* worker = new Thread(Scheduler::instance().getKernelProcess(), deadlineWorkerEntry, this,
                                nullptr, false, true, true);
    worker->setName("clock deadline worker");
    m_DeadlineWorker.adopt(worker);
    m_DeadlineMode = true;
    if (!worker->start()) {
      FATAL("RTC deadline worker could not be started.");
    }
    {
      LockGuard<Spinlock> guard(m_Lock);
      publishNextDeadlineLocked();
    }
    NOTICE("RTC: runtime IRQ8 disabled; CPU 0 LAPIC handles timer deadlines");
    return true;
  }
 
  IrqManager& irqManager = *Machine::instance().getIrqManager();
  m_IrqId = irqManager.registerIsaIrqHandler(8, this, IrqPolicy::levelThreaded());
  if (!m_IrqId) {
    return false;
  }
 
  const uint64_t baseline = getTickCountNano();
  m_TickCount = baseline;
  m_ProcessedTickCount = baseline;
  NOTICE("RTC: periodic interrupt frequency " << Dec << periodicIrqInfo[selectedIndex].Hz << " Hz");
  setPeriodicInterruptEnabled(true);
  return true;
}
 
void Rtc::synchronise(bool tohw) {
  const bool success = tohw ? writeHardwareClock() : readHardwareClock();
 
  if (!success) {
    if (tohw) {
      ERROR("RTC: hardware clock write failed (invalid date or timeout)");
    } else {
      ERROR("RTC: hardware clock read failed (invalid date or timeout)");
    }
  }
}
void Rtc::uninitialise() {
  if (m_DeadlineMode) {
    m_StopDeadlineWorker = 1;
    m_DeadlineScheduler->setClockDeadline(0);
    m_DeadlineScheduler->ringIrqWorkDoorbell(m_DeadlineWake);
    m_DeadlineWorker.join();
    m_DeadlineScheduler->unregisterWorkerWake(m_DeadlineWake);
    advanceCivilClock(getTickCountNano());
    m_DeadlineMode = 0;
  }
  setPeriodicInterruptEnabled(false);
  if (m_IrqId && !Machine::instance().getIrqManager()->unregisterHandler(m_IrqId, this)) {
    panic("RTC teardown could not drain its threaded IRQ callback");
  }
  m_IrqId = 0;
 
  synchronise(true);
 
  m_HandlerRegistry.reset();
 
  Alarm* reclaim = nullptr;
  Alarm* freeAlarms = nullptr;
  {
    LockGuard<Spinlock> guard(m_Lock);
    reclaim = m_AlarmQueue.detachActive();
    freeAlarms = m_AlarmQueue.detachFree();
  }
  while (reclaim) {
    Alarm* next = reclaim->next();
    delete reclaim;
    reclaim = next;
  }
  while (freeAlarms) {
    Alarm* next = freeAlarms->next();
    delete freeAlarms;
    freeAlarms = next;
  }
 
  // Free the I/O port range
  m_IoPort.free();
}
 
Rtc::Rtc()
    : m_IoPort("CMOS"),
      m_IrqId(0),
      m_PeriodicIrqInfoIndex(0),
      m_CenturyIndex(0),
      m_Year(1970),
      m_Month(1),
      m_DayOfMonth(1),
      m_Hour(0),
      m_Minute(0),
      m_Second(0),
      m_Nanosecond(0),
      m_TickCount(0),
      m_ProcessedTickCount(0),
      m_HandlerRegistry(),
      m_DeadlineScheduler(nullptr),
      m_DeadlineWorker(),
      m_DeadlineWaiters(),
      m_DeadlineWake(),
      m_DeadlinePending(0),
      m_StopDeadlineWorker(0),
      m_DeadlineMode(false),
      m_AlarmQueue(),
      m_Lock(false),
      m_CmosLock(),
      m_TscCalibration(),
      m_Tsc0(0),
      m_MonotonicTicks() {}
 
extern size_t g_FreePages;
extern size_t g_AllocedPages;
 
IrqDisposition Rtc::irq(irq_id_t number) {
  (void)number;
 
  uint8_t status = 0;
  {
    CmosTransactionGuard guard(m_CmosLock);
    status = readLocked(0x0C);
  }
 
  if (!(status & RtcInterruptRequested)) {
    return IrqDisposition::Handled;
  }
  if (!(status & RtcPeriodicFlag)) {
    return IrqDisposition::Handled;
  }
 
  processElapsedTime(getTickCountNano());
  return IrqDisposition::Handled;
}
 
void Rtc::processElapsedTime(uint64_t observed) {
  uint64_t elapsedSeconds = 0;
  const uint64_t delta = advanceCivilClock(observed, &elapsedSeconds);
  if (!delta) {
    return;
  }
 
  // Claim one due alarm under the queue lock, then publish it without
  // carrying that lock into Event or allocator code. A remover which sees
  // m_bDispatching waits for this ownership handoff to finish before it
  // returns and permits the Event to be reclaimed.
  while (true) {
    m_Lock.acquire();
    Alarm* claimed = m_AlarmQueue.claimDue(m_ProcessedTickCount, currentAlarmDispatchOwner());
    m_Lock.release();
 
    if (!claimed) {
      break;
    }
 
    Thread* target = reinterpret_cast<Thread*>(claimed->target());
    Event* event = reinterpret_cast<Event*>(claimed->event());
    target->sendEvent(event);
 
    m_Lock.acquire();
    m_AlarmQueue.completeDispatch(claimed);
    m_Lock.release();
  }
 
  if (UNLIKELY(delta >= Time::Multiplier::Millisecond ||
               m_Nanosecond >= Time::Multiplier::Millisecond)) {
    // Every millisecond, unblock any interrupts which were halted and halt
    // any which need to be halted.
    Machine::instance().getIrqManager()->tick();
  }
 
  if (UNLIKELY(elapsedSeconds)) {
#if MEMORY_LOGGING_ENABLED
    Serial* pSerial = Machine::instance().getSerial(1);
    NormalStaticString memoryLogStr;
    memoryLogStr += "Heap: ";
    memoryLogStr += (SlamAllocator::instance().heapPageCount() * TargetInfo::getPageSize()) / 1024;
    memoryLogStr += "K\tPages: ";
    memoryLogStr += (g_AllocedPages * TargetInfo::getPageSize()) / 1024;
    memoryLogStr += "K\t Free: ";
    memoryLogStr += (g_FreePages * TargetInfo::getPageSize()) / 1024;
    memoryLogStr += "K\n";
 
    pSerial->write_str(memoryLogStr);
 
    // Memory snapshot of current processes.
    for (size_t i = 0; i < Scheduler::instance().getNumProcesses(); ++i) {
      Scheduler::ProcessLease processLease;
      if (!Scheduler::instance().acquireProcess(processLease, i)) {
        continue;
      }
      Process* pProcess = processLease.get();
      LargeStaticString processListStr;
 
      ssize_t heapK = pProcess->getHeapUsage() / 1024;
      ssize_t virtK = (pProcess->getVirtualPageCount() * TargetInfo::getPageSize()) / 1024;
      ssize_t physK = (pProcess->getPhysicalPageCount() * TargetInfo::getPageSize()) / 1024;
      ssize_t shrK = (pProcess->getSharedPageCount() * TargetInfo::getPageSize()) / 1024;
 
      processListStr.append("\tProcess #");
      processListStr.append(pProcess->getId(), 10);
      processListStr.append(" '");
      processListStr.append(pProcess->description());
      processListStr.append("' V=");
      processListStr.append(virtK, 10);
      processListStr.append("K P=");
      processListStr.append(physK, 10);
      processListStr.append("K S=");
      processListStr.append(shrK, 10);
      processListStr.append("K Heap=");
      processListStr.append(heapK, 10);
      processListStr.append("K\n");
      pSerial->write_str(processListStr);
    }
#endif
  }
 
  // Timer delta is in nanoseconds.
  if (!m_DeadlineMode) {
    m_HandlerRegistry.dispatch(delta);
  }
}
 
uint64_t Rtc::advanceCivilClock(uint64_t observed, uint64_t* elapsedSeconds) {
  const uint64_t delta = RtcTimeAccounting::consumeElapsed(observed, m_ProcessedTickCount);
  if (!delta) {
    return 0;
  }
  m_TickCount = m_ProcessedTickCount;
  RtcTimeAccounting::CivilTime civilTime = {m_Year,   m_Month,  m_DayOfMonth, m_Hour,
                                            m_Minute, m_Second, m_Nanosecond};
  const uint64_t seconds = RtcTimeAccounting::advanceCivilTime(civilTime, delta);
  m_Year = civilTime.year;
  m_Month = civilTime.month;
  m_DayOfMonth = civilTime.day;
  m_Hour = civilTime.hour;
  m_Minute = civilTime.minute;
  m_Second = civilTime.second;
  m_Nanosecond = civilTime.nanosecond;
  if (elapsedSeconds) {
    *elapsedSeconds = seconds;
  }
  return delta;
}
 
void Rtc::setIndexLocked(uint8_t index) {
  uint8_t idx = m_IoPort.read8(0);
  m_IoPort.write8((idx & 0x80) | (index & 0x7F), 0);
}
 
bool Rtc::waitForUpdateCompletion(uint8_t index) {
  if (index > 9 && index != m_CenturyIndex) {
    return true;
  }
 
  const Time::Timestamp started = Time::getTicks();
  size_t polls = 0;
  while (true) {
    bool updating = false;
    {
      CmosTransactionGuard guard(m_CmosLock);
      setIndexLocked(0x0A);
      updating = (m_IoPort.read8(1) & 0x80) != 0;
    }
    if (!updating) {
      return true;
    }
    if (++polls >= RtcUpdateMaximumPolls || (Time::getTicks() - started) >= RtcUpdateTimeout) {
      ERROR(
          "RTC: timed out waiting for update completion before CMOS "
          "index "
          << index);
      return false;
    }
    Processor::pause();
  }
}
uint8_t Rtc::readCmos(uint8_t index) {
  return m_Instance.read(index);
}
 
uint8_t Rtc::read(uint8_t index) {
  uint8_t value = 0xFF;
  (void)read(index, value);
  return value;
}
 
bool Rtc::read(uint8_t index, uint8_t& value) {
  if (!waitForUpdateCompletion(index)) {
    return false;
  }
  CmosTransactionGuard guard(m_CmosLock);
  value = readLocked(index);
  return true;
}
 
uint8_t Rtc::readLocked(uint8_t index) {
  setIndexLocked(index);
  return m_IoPort.read8(1);
}
 
bool Rtc::write(uint8_t index, uint8_t value) {
  if (!waitForUpdateCompletion(index)) {
    return false;
  }
 
  CmosTransactionGuard guard(m_CmosLock);
  writeLocked(index, value);
  return true;
}
 
void Rtc::writeCmos(uint8_t index, uint8_t value) {
  (void)m_Instance.write(index, value);
}
 
bool Rtc::inhibitClockUpdatesLocked(uint8_t& status) {
  // UIP's quiet window is only guaranteed for 244 us. Do not allow a
  // scheduler preemption between the final check and freezing the clock.
  const bool interrupts = Processor::getInterrupts();
  Processor::setInterrupts(false);
  if (readLocked(0x0A) & 0x80) {
    Processor::setInterrupts(interrupts);
    return false;
  }
  status = readLocked(0x0B);
  writeLocked(0x0B, status | RtcUpdateInhibit);
  const bool inhibited = (readLocked(0x0B) & RtcUpdateInhibit) != 0;
  Processor::setInterrupts(interrupts);
  return inhibited;
}
 
bool Rtc::readHardwareClock() {
  if (!waitForUpdateCompletion(0)) {
    return false;
  }
  CmosTransactionGuard guard(m_CmosLock);
  uint8_t status = 0;
  if (!inhibitClockUpdatesLocked(status)) {
    return false;
  }
  RtcTimeAccounting::CivilTime time = {};
  const bool success =
      RtcCalendar::read([this](uint8_t index) { return readLocked(index); }, m_CenturyIndex, time);
  writeLocked(0x0B, status);
  if (!success) {
    return false;
  }
  m_Second = time.second;
  m_Minute = time.minute;
  m_Hour = time.hour;
  m_DayOfMonth = time.day;
  m_Month = time.month;
  m_Year = time.year;
  return true;
}
 
bool Rtc::writeHardwareClock() {
  const RtcTimeAccounting::CivilTime time = {m_Year,   m_Month, m_DayOfMonth, m_Hour, m_Minute,
                                             m_Second, 0};
  if (!RtcCalendar::valid(time) || (!m_CenturyIndex && time.year > 2069) ||
      !waitForUpdateCompletion(0)) {
    return false;
  }
  CmosTransactionGuard guard(m_CmosLock);
  uint8_t status = 0;
  if (!inhibitClockUpdatesLocked(status)) {
    return false;
  }
  const bool success = RtcCalendar::write(
      [this](uint8_t index) { return readLocked(index); },
      [this](uint8_t index, uint8_t value) { writeLocked(index, value); }, m_CenturyIndex, time);
  writeLocked(0x0B, status);
  return success;
}
 
void Rtc::writeLocked(uint8_t index, uint8_t value) {
  setIndexLocked(index);
  m_IoPort.write8(value, 1);
}
 
void Rtc::setPeriodicInterruptEnabled(bool enabled) {
  CmosTransactionGuard guard(m_CmosLock);
  const uint8_t status = readLocked(0x0B);
  const uint8_t quiescentStatus = static_cast<uint8_t>(status & ~RtcInterruptEnableMask);
  writeLocked(0x0B, enabled ? static_cast<uint8_t>(quiescentStatus | RtcPeriodicInterruptEnable)
                            : quiescentStatus);
 
  // Some RTCs require register C to be cleared after control changes.
  (void)readLocked(0x0C);
}