system/kernel/core/main.cc

/*
 * Copyright (c) 2008-2014, Pedigree Developers
 *
 * Please see the CONTRIB file in the root of the source tree for a full
 * list of contributors.
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */
 
#include "pedigree/kernel/Archive.h"
#include "pedigree/kernel/Atomic.h"
#include "pedigree/kernel/BootstrapInfo.h"
#include "pedigree/kernel/Log.h"
#include "pedigree/kernel/ServiceManager.h"
#include "pedigree/kernel/Subsystem.h"
#include "pedigree/kernel/Version.h"
#include "pedigree/kernel/compiler.h"
#include "pedigree/kernel/core/BootIO.h"
#include "pedigree/kernel/core/SlamAllocator.h"
#include "pedigree/kernel/core/cppsupport.h"
#include "pedigree/kernel/graphics/GraphicsService.h"
#include "pedigree/kernel/linker/KernelElf.h"
#include "pedigree/kernel/machine/InputManager.h"
#include "pedigree/kernel/machine/Machine.h"
#include "pedigree/kernel/machine/Trace.h"
#include "pedigree/kernel/panic.h"
#include "pedigree/kernel/process/InfoBlock.h"
#include "pedigree/kernel/process/MemoryPressureKiller.h"
#include "pedigree/kernel/process/MemoryPressureManager.h"
#include "pedigree/kernel/process/OwnedThread.h"
#include "pedigree/kernel/process/PerProcessorScheduler.h"
#include "pedigree/kernel/process/Process.h"
#include "pedigree/kernel/process/Scheduler.h"
#include "pedigree/kernel/process/Thread.h"
#include "pedigree/kernel/processor/KernelCoreSyscallManager.h"
#include "pedigree/kernel/processor/PhysicalMemoryManager.h"
#include "pedigree/kernel/processor/Processor.h"
#include "pedigree/kernel/processor/ProcessorInformation.h"
#include "pedigree/kernel/processor/types.h"
#include "pedigree/kernel/utilities/Cache.h"
#include "pedigree/kernel/utilities/StaticString.h"
#include "pedigree/kernel/utilities/String.h"
#include "pedigree/kernel/utilities/new"
#include "pedigree/kernel/utilities/utility.h"
 
#if DEBUGGER
#include "pedigree/kernel/debugger/Debugger.h"
#include "pedigree/kernel/debugger/commands/LocksCommand.h"
#endif
 
#if THREADS
#include "pedigree/kernel/utilities/ZombieQueue.h"
#endif
 
#if HOSTED
namespace __pedigree_hosted {};  // namespace __pedigree_hosted
using namespace __pedigree_hosted;
#include <stdio.h>
#endif
 
EXPORTED_PUBLIC BootIO bootIO;
 
BootstrapStruct_t* g_pBootstrapInfo;
 
static Atomic<bool> g_NeedsShutdown(false);
static Machine::ShutdownType g_ShutdownType = Machine::ShutdownType::Halt;
static Atomic<bool> g_ShutdownCoordinator(false);
 
#if HOSTED && PEDIGREE_HOSTED_IRQ_CLOSURE_TESTS
extern bool runHostedIrqClosureRegressions();
#endif
 
class SlamRecovery : public MemoryPressureHandler {
 public:
  virtual const char* getMemoryPressureDescription();
  virtual bool compact();
};
 
#if MULTIPROCESSOR
void apMain() {
  NOTICE("Processor #" << Processor::id() << " started.");
 
  EMIT_IF(THREADS) {
    // Add us as the idle thread for this CPU.
    Processor::information().getScheduler().setIdle(Processor::information().getCurrentThread());
  }
 
  for (;;) {
#if THREADS
    Processor::information().getScheduler().idleUntilInterrupt();
    Scheduler::instance().yield();
#else
    Processor::haltUntilInterrupt();
#endif
  }
}
#endif
 
ModuleInfo* g_StaticDrivers[128];
size_t g_StaticDriverN = 0;
 
static int loadModules(void* inf) {
  Archive* initrd = nullptr;
 
  EMIT_IF(STATIC_DRIVERS) {
    extern uintptr_t start_module_ctors;
    extern uintptr_t end_module_ctors;
 
    // Call static constructors before we start. If we don't... there won't
    // be any properly initialised ModuleInfo structures :)
    uintptr_t* iterator = &start_module_ctors;
    while (iterator < &end_module_ctors) {
      void (*fp)(void) = reinterpret_cast<void (*)(void)>(*iterator);
      fp();
      iterator++;
    }
 
    for (size_t i = 0; i < g_StaticDriverN; ++i) {
      assert(g_StaticDrivers[i]->tag == MODULE_TAG);
      KernelElf::instance().loadModule(g_StaticDrivers[i]);
    }
 
    KernelElf::instance().executeModules();
  }
  else {
    BootstrapStruct_t* bsInf = static_cast<BootstrapStruct_t*>(inf);
 
    NOTICE("initrd @ " << Hex << bsInf->getInitrdAddress() << " -> "
                       << (bsInf->getInitrdAddress() + bsInf->getInitrdSize()) << ", " << Dec
                       << bsInf->getInitrdSize() << " bytes");
 
    initrd = new Archive(bsInf->getInitrdAddress(), bsInf->getInitrdSize());
    bsInf = nullptr;
 
    size_t nFiles = initrd->getNumFiles();
    NOTICE("there are " << nFiles << " files");
    g_BootProgressTotal = nFiles * 2;  // Each file has to be preloaded and executed.
    for (size_t i = 0; i < nFiles; i++) {
      // Handle archives with `._<filename>` entries from Apple tar file
      // creation
      if (!StringCompareN(initrd->getFileName(i), "._", 2)) {
        continue;
      }
 
      NOTICE("loading module #" << i << " (" << initrd->getFileName(i) << ")...");
      Processor::setInterrupts(true);
      KernelElf::instance().loadModule(reinterpret_cast<uint8_t*>(initrd->getFile(i)),
                                       initrd->getFileSize(i));
      if (!Processor::getInterrupts())
        WARNING("A loaded module disabled interrupts.");
    }
 
    // Start any modules we can run already.
    KernelElf::instance().executeModules();
  }
 
  // Wait for all modules to finish loading before we continue.
  KernelElf::instance().waitForModulesToLoad();
 
  // The initialisation is done here, unmap/free the .init section and on
  // x86/64 the identity mapping of 0-4MB NOTE: BootstrapStruct_t unusable
  // after this point
  Processor::initialisationDone();
 
#if PEDIGREE_CONCURRENCY_SMOKE_TESTS
  // The ISO-only smoke image deliberately has no root filesystem. Once every
  // eligible module has retired, request a controlled shutdown instead of
  // invoking userspace through the intentionally pending POSIX dependency.
  g_NeedsShutdown = true;
#else
  // Now that we've cleaned up and are done loading modules, we can run the
  // init module.
  KernelElf::instance().invokeInitModule();
 
  if (KernelElf::instance().hasPendingModules()) {
    FATAL("At least one module's dependencies were never met.");
  }
#endif
 
  // It's now safe to clean up the initrd archive
  if (initrd) {
    delete initrd;
  }
 
#if HOSTED
  fprintf(stderr, "Pedigree has started: all modules have been loaded.\n");
#endif
 
  NOTICE("module load thread is terminating");
  return 0;
}
 
extern "C" void _main(BootstrapStruct_t& bsInf) USED;
void _cxx_main(BootstrapStruct_t& bsInf);
extern "C" void _main(BootstrapStruct_t& bsInf) {
  _cxx_main(bsInf);
}
 
void _cxx_main(BootstrapStruct_t& bsInf) {
  TRACE("constructors");
 
  // Firstly call the constructors of all global objects.
  initialiseConstructors();
 
  g_pBootstrapInfo = &bsInf;
 
#if DEBUGGER && TRACK_LOCKS
  g_LocksCommand.setReady();
#endif
 
  TRACE("Processor init");
 
  // Initialise the processor-specific interface
  Processor::initialise1(bsInf);
 
  TRACE("log init");
 
  // Initialise the kernel log
  Log::instance().initialise1();
 
  TRACE("Machine init");
 
  // Initialise the machine-specific interface
  Machine& machine = Machine::instance();
  Machine::instance().initialiseDeviceTree();
 
  machine.initialise();
 
#if DEBUGGER && (!defined(PEDIGREE_HOSTED_DARWIN) || !PEDIGREE_HOSTED_DARWIN)
  TRACE("Debugger init");
  Debugger::instance().initialise();
#endif
 
  TRACE("Machine init2");
 
  machine.initialise2();
 
  TRACE("Log init2");
 
  // Initialise the kernel log's callbacks
  Log::instance().initialise2();
 
  TRACE("Processor init2");
 
  // Initialise the processor-specific interface
  // Bootup of the other Application Processors and related tasks
  Processor::initialise2(bsInf);
 
  TRACE("Machine init3");
 
  machine.initialise3();
 
  TRACE("KernelElf init");
 
  // Initialise the Kernel Elf class
  if (KernelElf::instance().initialise(bsInf) == false)
    panic("KernelElf::initialise() failed");
 
  EMIT_IF(!STATIC_DRIVERS) {
    // initrd needed if drivers aren't statically linked.
    if (bsInf.isInitrdLoaded() == false)
      panic("Initrd module not loaded!");
  }
 
  TRACE("kernel syscall init");
 
  KernelCoreSyscallManager::instance().initialise();
 
  TRACE("initial init done, enabling interrupts");
 
  Processor::setInterrupts(true);
 
  TRACE("bootIO init");
 
  // Initialise the boot output.
  bootIO.initialise();
 
  // Spew out a starting string.
  HugeStaticString str, ident;
  str += "Pedigree - revision ";
  str += g_pBuildRevision;
  EMIT_IF(DONT_LOG_TO_SERIAL) {
    str += "\n=======================\n";
  }
  else {
    str += "\r\n=======================\r\n";
  }
  bootIO.write(str, BootIO::White, BootIO::Black);
 
  str.clear();
  str += "Built at ";
  str += g_pBuildTime;
  str += " by ";
  str += g_pBuildUser;
  str += " on ";
  str += g_pBuildMachine;
  EMIT_IF(DONT_LOG_TO_SERIAL) {
    str += "\n";
  }
  else {
    str += "\r\n";
  }
  bootIO.write(str, BootIO::LightGrey, BootIO::Black);
 
  str.clear();
  str += "Build flags: ";
  str += g_pBuildFlags;
  EMIT_IF(DONT_LOG_TO_SERIAL) {
    str += "\n";
  }
  else {
    str += "\r\n";
  }
  bootIO.write(str, BootIO::LightGrey, BootIO::Black);
 
  str.clear();
  str += "Processor information: ";
  Processor::identify(ident);
  str += ident;
  EMIT_IF(DONT_LOG_TO_SERIAL) {
    str += "\n";
  }
  else {
    str += "\r\n";
  }
  bootIO.write(str, BootIO::LightGrey, BootIO::Black);
 
  TRACE("creating graphics service");
 
  // Set up the graphics service for drivers to register with
  EMIT_IF(!NOGFX) {
    GraphicsService* pService = new GraphicsService;
    ServiceFeatures* pFeatures = new ServiceFeatures;
    pFeatures->add(ServiceFeatures::touch);
    pFeatures->add(ServiceFeatures::withdraw);
    pFeatures->add(ServiceFeatures::probe);
    ServiceManager::instance().addService(String("graphics"), pService, pFeatures);
  }
 
  TRACE("creating memory pressure handlers");
 
  // Set up SLAM recovery memory pressure handler.
  SlamRecovery recovery;
  MemoryPressureManager::instance().registerHandler(MemoryPressureManager::HighestPriority,
                                                    &recovery);
 
  // Set up the process killer memory pressure handler.
  MemoryPressureProcessKiller killer;
  MemoryPressureManager::instance().registerHandler(MemoryPressureManager::LowestPriority, &killer);
 
  // Set up the global info block manager.
  TRACE("InfoBlockManager init");
  InfoBlockManager::instance().initialise();
 
  // Bring up the cache subsystem.
  TRACE("CacheManager init");
  CacheManager::instance().initialise();
 
  // Initialise the input manager
  TRACE("InputManager init");
  InputManager::instance().initialise();
 
  EMIT_IF(THREADS) {
    TRACE("ZombieQueue init");
    ZombieQueue::instance().initialise();
  }
 
 
  OwnedThread moduleLoadThread;
#if HOSTED && PEDIGREE_HOSTED_IRQ_CLOSURE_TESTS
  runHostedIrqClosureRegressions();
#else
  TRACE("starting module load thread");
 
  EMIT_IF(THREADS) {
    Thread* pThread = new Thread(Processor::information().getCurrentThread()->getParent(),
                                 &loadModules, static_cast<void*>(&bsInf), 0);
    pThread->setName("module load thread");
    moduleLoadThread.adopt(pThread);
  }
  else {
    loadModules(&bsInf);
  }
#endif
 
  EMIT_IF(DEBUGGER_RUN_AT_START) {
    Processor::breakpoint();
  }
 
  TRACE("becoming idle");
 
  EMIT_IF(THREADS) {
    // Add us as the idle thread for this CPU.
    Processor::information().getScheduler().setIdle(Processor::information().getCurrentThread());
  }
 
  // This will run when nothing else is available to run
  while (!g_NeedsShutdown) {
#if THREADS
    Processor::information().getScheduler().idleUntilInterrupt();
#else
    Processor::haltUntilInterrupt();
#endif
 
    // A shutdown wake can resume this halt before the next predicate check.
    // Do not yield back to a busy ready queue before retiring the idle role.
    if (!g_NeedsShutdown)
      Scheduler::instance().yield();
  }
 
  EMIT_IF(THREADS) {
    // Shutdown joins need this Thread to enter an ordinary WaitQueue and
    // be republished when a worker exits. An idle Thread is only a fallback
    // and never commits Sleeping, so retire that role before the first
    // join rather than leaving its wake behind unrelated ready workers.
    Processor::information().getScheduler().setIdle(nullptr);
 
    // A module can request shutdown from its entry point. The loader still
    // has bookkeeping and initrd cleanup to finish after that request, so
    // module teardown must not race it.
    moduleLoadThread.join();
#if HOSTED && PEDIGREE_HOSTED_SMOKE_TESTS && !PEDIGREE_HOSTED_IRQ_CLOSURE_TESTS
    NOTICE("HOSTED-WAIT-TEST: PASS module-loader-shutdown-join");
#endif
  }
 
  NOTICE("Resetting...");
 
  // Clean up all loaded modules (unmounts filesystems and the like).
  Machine::setShutdownPhase(Machine::ShutdownPhase::Userspace);
  if (!KernelElf::instance().unloadModules()) {
    panic("Shutdown aborted: module quiesce failed");
  }
 
  Machine::setShutdownPhase(Machine::ShutdownPhase::Firmware);
  if (!Machine::instance().prepareShutdown(g_ShutdownType)) {
    panic("Shutdown aborted: firmware workers did not stop");
  }
 
  EMIT_IF(STATIC_DRIVERS) {
    extern uintptr_t start_module_dtors;
    extern uintptr_t end_module_dtors;
 
    // Call all the module destructors now
    uintptr_t* iterator = &start_module_dtors;
    while (iterator < &end_module_dtors) {
      void (*fp)(void) = reinterpret_cast<void (*)(void)>(*iterator);
      fp();
      iterator++;
    }
  }
 
  EMIT_IF(THREADS) {
    // Module exits may publish mandatory deferred destruction. Keep the
    // worker alive through every exit, then join it before lower-level
    // scheduler and interrupt teardown begins.
    Machine::setShutdownPhase(Machine::ShutdownPhase::Destructors);
    ZombieQueue::instance().destroy();
  }
 
  // No need for user input anymore.
  Machine::setShutdownPhase(Machine::ShutdownPhase::Input);
  InputManager::instance().shutdown();
 
  // Pinned modules can retain caches and page loans through the final handoff.
  // Their backends must remain available for this last synchronous writeback.
  Machine::setShutdownPhase(Machine::ShutdownPhase::Caches);
#if HOSTED
  CacheManager::destroyInstance();
#else
  if (!CacheManager::instance().shutdown()) {
    panic("Shutdown aborted: retained cache writeback failed");
  }
  NOTICE("CacheManager: terminal cache writeback complete");
#endif
 
  // The shared info block remains live until userspace and module teardown
  // finish, but its callback must retire before the platform timer does.
  Machine::setShutdownPhase(Machine::ShutdownPhase::Timers);
  if (!InfoBlockManager::instance().shutdown()) {
    panic("Shutdown aborted: info block timer callback did not drain");
  }
#if PEDIGREE_CONCURRENCY_SMOKE_TESTS
  NOTICE("QEMU-CONCURRENCY-TEST: PASS infoblock-timer-drain");
#endif
 
  // Stop active platform services while their worker/callback drains can
  // still schedule.
  Machine::setShutdownPhase(Machine::ShutdownPhase::Devices);
  Machine::instance().deinitialise();
 
  // Teardown above joins module, input, keyboard, and threaded-IRQ workers.
  // Keep every scheduler CPU available until those drains have completed;
  // once the remaining work is strictly local, retire the APs permanently.
  Machine::setShutdownPhase(Machine::ShutdownPhase::Processors);
  EMIT_IF(MULTIPROCESSOR) {
    if (!Machine::instance().stopAllOtherProcessors()) {
      ERROR_NOLOCK("Shutdown aborted: not all other processors stopped");
      Processor::setInterrupts(false);
      while (true)
        Processor::halt();
    }
  }
 
#if PEDIGREE_CONCURRENCY_SMOKE_TESTS
  NOTICE("QEMU-CONCURRENCY-TEST: PASS shutdown-worker-drain-smp");
#endif
 
  Processor::setInterrupts(false);
 
#if !HOSTED
  // Firmware register access can still allocate mappings. Keep physical-memory
  // bookkeeping alive until the final attempt to power off or reset returns.
  Machine::setShutdownPhase(Machine::ShutdownPhase::FinalAction);
  StaticString<256> shutdownResult;
  if (const char* result = Machine::instance().finalShutdown(g_ShutdownType)) {
    shutdownResult = result;
  }
#endif
 
  // Release processor bookkeeping before hosted global destruction or after a
  // bare-metal firmware action returned without switching the machine off.
#if HOSTED
  Machine::setShutdownPhase(Machine::ShutdownPhase::ProcessorCleanup);
#else
  // Keep the firmware result visible even if processor cleanup itself fails.
  Machine::setShutdownPhase(Machine::ShutdownPhase::ProcessorCleanup, shutdownResult);
#endif
  Processor::deinitialise();
 
#if HOSTED
  NOTICE("Module shutdown complete. Running destructors and terminating...");
  runKernelDestructors();
#else
  // Bare-metal ends in a terminal halt loop, not an owning runtime.
  // Retain static storage because its providers are already quiesced and the
  // active kernel page tables cannot be destroyed while still executing.
  NOTICE("Module shutdown complete. Retaining kernel static storage and terminating...");
#if PEDIGREE_CONCURRENCY_SMOKE_TESTS
  TRACE("QEMU-CONCURRENCY-TEST: PASS kernel-static-storage-retained");
#endif
#endif
 
  // Hosted returns to its owning runtime after the terminal trace.
  TRACE("kernel main() terminating");
 
#if !HOSTED
  StaticString<512> shutdownMessage(
      "Shutdown complete.\nIt's now safe to power off your computer.");
  if (shutdownResult.length()) {
    shutdownMessage += "\n\n";
    shutdownMessage += shutdownResult;
  }
  Machine::instance().displayShutdownMessage(shutdownMessage);
 
  // An NMI may wake HLT; return to the halted state with IRQs disabled.
  while (true) {
    Processor::setInterrupts(false);
    Processor::halt();
  }
#endif
}
 
void EXPORTED_PUBLIC system_reset();
void system_reset() {
  // Close out the main thread even when user work keeps the CPU busy.
  g_NeedsShutdown = true;
  EMIT_IF(THREADS) {
    if (auto* scheduler = Scheduler::instance().getBootstrapProcessorScheduler())
      scheduler->requestIdleThreadWakeup();
  }
}
 
void system_reboot(Machine::ShutdownType type) {
  WARNING("System shutting down...");
  Process* currentProcess = Processor::information().getCurrentThread()->getParent();
 
  Subsystem* currentSubsystem = currentProcess->getSubsystem();
  if (!currentSubsystem) {
    panic("System reboot requires a userspace shutdown coordinator");
  }
 
  // Concurrent requests must not change the selected terminal action.
  if (!g_ShutdownCoordinator.compareAndSwap(false, true)) {
    currentSubsystem->exit(0);
    panic("Concurrent shutdown caller returned from process exit");
  }
 
  // The caller may still be a shell child. Its parent, orphan exit path,
  // or terminal process drain claims the reaper after ordinary termination.
  g_ShutdownType = type;
  Machine::setShutdownPhase(Machine::ShutdownPhase::Requested);
  system_reset();
  // Exit can block while main starts teardown and retires its idle role.
  // Use ordinary scheduling instead of reserving a later handoff to that role.
  currentSubsystem->exit(0);
  panic("Shutdown coordinator returned from process exit");
}
 
const char* SlamRecovery::getMemoryPressureDescription() {
  return "SLAM recovery; freeing unused slabs.";
}
 
bool SlamRecovery::compact() {
  return SlamAllocator::instance().recovery(5) != 0;
}