loader.c

/*
 * Minimal UEFI loader for the native x86_64 kernel.
 *
 * This file intentionally uses only UEFI protocols and freestanding C. The
 * PE/COFF wrapper is generated by scripts/uefi_pe.py so the loader remains
 * buildable with the host clang without adding a second C runtime.
 */
 
#include <stdint.h>
 
#include "exit_boot_services.h"
#include "framebuffer.h"
#include "load_options.h"
 
typedef uint16_t efi_char16_t;
typedef struct efi_guid {
  uint32_t a;
  uint16_t b;
  uint16_t c;
  uint8_t d[8];
} efi_guid_t;
 
typedef struct efi_system_table efi_system_table_t;
typedef struct efi_boot_services efi_boot_services_t;
typedef struct efi_file efi_file_t;
 
typedef efi_status_t (*efi_handle_protocol_t)(efi_handle_t, efi_guid_t*, void**);
typedef efi_status_t (*efi_allocate_pages_t)(uint32_t, uint32_t, uint64_t, uint64_t*);
typedef efi_status_t (*efi_locate_device_path_t)(efi_guid_t*, void**, efi_handle_t*);
typedef efi_status_t (*efi_locate_handle_buffer_t)(uint32_t, efi_guid_t*, void*, uint64_t*,
                                                   efi_handle_t**);
typedef efi_status_t (*efi_locate_protocol_t)(efi_guid_t*, void*, void**);
 
struct efi_boot_services {
  uint8_t header[24];
  void* raise_tpl;
  void* restore_tpl;
  void* allocate_pages;
  void* free_pages;
  void* get_memory_map;
  void* allocate_pool;
  efi_free_pool_t free_pool;
  uint8_t before_handle_protocol[9 * 8];
  efi_handle_protocol_t handle_protocol;
  void* reserved_after_handle_protocol;
  void* register_protocol_notify;
  void* locate_handle;
  efi_locate_device_path_t locate_device_path;
  void* install_configuration_table;
  void* load_image;
  void* start_image;
  void* exit;
  void* unload_image;
  efi_exit_boot_services_t exit_boot_services;
  uint8_t between_exit_and_open[5 * 8];
  void* open_protocol;
  void* close_protocol;
  void* open_protocol_information;
  void* protocols_per_handle;
  efi_locate_handle_buffer_t locate_handle_buffer;
  efi_locate_protocol_t locate_protocol;
};
 
typedef efi_status_t (*efi_output_string_t)(void*, efi_char16_t*);
typedef struct efi_simple_text_output {
  void* reset;
  efi_output_string_t output_string;
} efi_simple_text_output_t;
 
struct efi_system_table {
  uint8_t header[24];
  efi_char16_t* firmware_vendor;
  uint32_t firmware_revision;
  efi_handle_t console_in_handle;
  void* con_in;
  efi_handle_t console_out_handle;
  efi_simple_text_output_t* con_out;
  efi_handle_t standard_error_handle;
  void* std_err;
  void* runtime_services;
  efi_boot_services_t* boot_services;
  uint64_t configuration_table_entries;
  void* configuration_table;
};
 
typedef efi_status_t (*efi_file_open_t)(efi_file_t*, efi_file_t**, efi_char16_t*, uint64_t,
                                        uint64_t);
typedef efi_status_t (*efi_file_close_t)(efi_file_t*);
typedef efi_status_t (*efi_file_read_t)(efi_file_t*, uint64_t*, void*);
typedef efi_status_t (*efi_file_get_info_t)(efi_file_t*, efi_guid_t*, uint64_t*, void*);
 
struct efi_file {
  uint64_t revision;
  efi_file_open_t open;
  efi_file_close_t close;
  void* delete_file;
  efi_file_read_t read;
  void* write;
  void* get_position;
  void* set_position;
  efi_file_get_info_t get_info;
};
 
typedef struct efi_simple_file_system {
  uint64_t revision;
  efi_status_t (*open_volume)(struct efi_simple_file_system*, efi_file_t**);
} efi_simple_file_system_t;
 
typedef struct efi_loaded_image {
  uint32_t revision;
  uint32_t reserved;
  efi_handle_t parent_handle;
  efi_system_table_t* system_table;
  efi_handle_t device_handle;
  void* file_path;
  void* reserved2;
  uint32_t load_options_size;
  void* load_options;
} efi_loaded_image_t;
 
typedef struct efi_device_path {
  uint8_t type;
  uint8_t subtype;
  uint16_t length;
} __attribute__((packed)) efi_device_path_t;
 
typedef struct efi_configuration_table {
  efi_guid_t vendor_guid;
  void* vendor_table;
} efi_configuration_table_t;
 
typedef struct bootstrap_module {
  uint64_t base;
  uint64_t end;
  uint64_t name;
  uint64_t pad;
} bootstrap_module_t;
 
typedef struct bootstrap_info {
  uint32_t flags;
  uint32_t reserved;
  uint32_t module_count;
  uint64_t modules;
  uint32_t section_count;
  uint32_t section_entry_size;
  uint32_t section_string_index;
  uint64_t section_headers;
  uint64_t memory_map;
  uint32_t memory_map_length;
  uint32_t memory_map_entry_size;
  uint64_t command_line;
  uint64_t acpi_rsdp;
  uint64_t smbios;
  uint64_t framebuffer;
  uint32_t framebuffer_width;
  uint32_t framebuffer_height;
  uint32_t framebuffer_pitch;
  uint32_t framebuffer_bpp;
  uint32_t framebuffer_format;
} __attribute__((packed)) bootstrap_info_t;
 
_Static_assert(sizeof(bootstrap_info_t) == 108, "bootstrap wire layout must match the kernel");
 
typedef struct elf64_header {
  uint8_t ident[16];
  uint16_t type;
  uint16_t machine;
  uint32_t version;
  uint64_t entry;
  uint64_t program_header_offset;
  uint64_t section_header_offset;
  uint32_t flags;
  uint16_t header_size;
  uint16_t program_header_size;
  uint16_t program_header_count;
  uint16_t section_header_size;
  uint16_t section_header_count;
  uint16_t section_string_index;
} __attribute__((packed)) elf64_header_t;
 
typedef struct elf64_program_header {
  uint32_t type;
  uint32_t flags;
  uint64_t offset;
  uint64_t virtual_address;
  uint64_t physical_address;
  uint64_t file_size;
  uint64_t memory_size;
  uint64_t alignment;
} __attribute__((packed)) elf64_program_header_t;
 
typedef struct elf64_section_header {
  uint32_t name;
  uint32_t type;
  uint64_t flags;
  uint64_t address;
  uint64_t offset;
  uint64_t size;
  uint32_t link;
  uint32_t info;
  uint64_t alignment;
  uint64_t entry_size;
} __attribute__((packed)) elf64_section_header_t;
 
#define EFI_ALLOCATE_MAX_ADDRESS 1
#define EFI_ALLOCATE_ADDRESS 2
#define EFI_LOADER_DATA 4
#define EFI_FILE_MODE_READ 1
#define EFI_CONVENTIONAL_MEMORY 7
#define EFI_ACPI_RECLAIM_MEMORY 9
#define EFI_ACPI_MEMORY_NVS 10
#define EFI_LOCATE_BY_PROTOCOL 2
#define EFI_DEVICE_PATH_TYPE_END 0x7f
#define EFI_DEVICE_PATH_TYPE_MEDIA 4
#define EFI_DEVICE_PATH_SUBTYPE_FILE_PATH 4
#define BOOTSTRAP_FLAG_CMDLINE 0x001
#define BOOTSTRAP_FLAG_UEFI 0x080
#define EFI_PAGE_SIZE 4096
#define BOOTSTRAP_FLAG_MODULES 0x002
#define BOOTSTRAP_FLAG_ELF 0x004
#define BOOTSTRAP_FLAG_MEMORY_MAP 0x008
#define BOOTSTRAP_FLAG_ACPI 0x010
#define BOOTSTRAP_FLAG_SMBIOS 0x020
#define BOOTSTRAP_FLAG_FRAMEBUFFER 0x040
#define PHYS_ALIAS 0xffff800000000000ULL
#define KERNEL_BASE 0xffffffff7ff00000ULL
#define PT_LOAD 1
#define SHF_ALLOC 2
 
static const efi_guid_t loaded_image_guid = {
    0x5b1b31a1, 0x9562, 0x11d2, {0x8e, 0x3f, 0x00, 0xa0, 0xc9, 0x69, 0x72, 0x3b}};
static const efi_guid_t simple_file_system_guid = {
    0x964e5b22, 0x6459, 0x11d2, {0x8e, 0x39, 0x00, 0xa0, 0xc9, 0x69, 0x72, 0x3b}};
static const efi_guid_t file_info_guid = {
    0x09576e92, 0x6d3f, 0x11d2, {0x8e, 0x39, 0x00, 0xa0, 0xc9, 0x69, 0x72, 0x3b}};
static const efi_guid_t acpi20_guid = {
    0x8868e871, 0xe4f1, 0x11d3, {0xbc, 0x22, 0x00, 0x80, 0xc7, 0x3c, 0x88, 0x81}};
static const efi_guid_t smbios_guid = {
    0xeb9d2d31, 0x2d88, 0x11d3, {0x9a, 0x16, 0x00, 0x90, 0x27, 0x3f, 0xc1, 0x4d}};
static const efi_guid_t graphics_output_guid = {
    0x9042a9de, 0x23dc, 0x4a38, {0x96, 0xfb, 0x7a, 0xde, 0xd0, 0x80, 0x51, 0x6a}};
static const efi_guid_t edid_active_guid = {
    0xbd8c1056, 0x9f36, 0x44ec, {0x92, 0xa8, 0xa6, 0x33, 0x7f, 0x81, 0x79, 0x86}};
static const efi_guid_t device_path_guid = {
    0x09576e91, 0x6d3f, 0x11d2, {0x8e, 0x39, 0x00, 0xa0, 0xc9, 0x69, 0x72, 0x3b}};
static const efi_char16_t current_prefix[] = L"\\EFI\\PEDIGREE\\current\\";
static const efi_char16_t known_good_prefix[] = L"\\EFI\\PEDIGREE\\known-good\\";
static const efi_char16_t debug_prefix[] = L"\\EFI\\PEDIGREE\\debug\\";
static const efi_char16_t removable_prefix[] = L"\\EFI\\BOOT\\";
 
static efi_system_table_t* g_system_table;
 
static efi_edid_active_t* framebuffer_edid(efi_handle_t graphics_handle) {
  efi_boot_services_t* services = g_system_table->boot_services;
  efi_edid_active_t* edid = 0;
  if (services->handle_protocol(graphics_handle, (efi_guid_t*)&edid_active_guid, (void**)&edid) ==
          EFI_SUCCESS &&
      edid)
    return edid;
  // EDID may be attached to an output child below the framebuffer's GOP handle.
  efi_handle_t* outputs = 0;
  uint64_t count = 0;
  edid = 0;
  if (services->locate_handle_buffer(EFI_LOCATE_BY_PROTOCOL, (efi_guid_t*)&edid_active_guid, 0,
                                     &count, &outputs) == EFI_SUCCESS &&
      outputs) {
    for (uint64_t i = 0; i < count && !edid; ++i) {
      void* path = 0;
      efi_handle_t owner = 0;
      if (services->handle_protocol(outputs[i], (efi_guid_t*)&device_path_guid, &path) !=
              EFI_SUCCESS ||
          !path ||
          services->locate_device_path((efi_guid_t*)&graphics_output_guid, &path, &owner) !=
              EFI_SUCCESS ||
          owner != graphics_handle)
        continue;
      if (services->handle_protocol(outputs[i], (efi_guid_t*)&edid_active_guid, (void**)&edid) !=
          EFI_SUCCESS)
        edid = 0;
    }
  }
  if (outputs)
    services->free_pool(outputs);
  return edid;
}
 
static void prepare_framebuffer(bootstrap_info_t* info) {
  efi_boot_services_t* services = g_system_table->boot_services;
  efi_graphics_output_t* console = 0;
  boot_framebuffer_t framebuffer;
  if (services->handle_protocol(g_system_table->console_out_handle,
                                (efi_guid_t*)&graphics_output_guid,
                                (void**)&console) != EFI_SUCCESS)
    console = 0;
  const uint64_t console_address = console && console->mode ? console->mode->framebuffer : 0;
  efi_handle_t* handles = 0;
  uint64_t count = 0;
  int selected = 0;
  if (services->locate_handle_buffer(EFI_LOCATE_BY_PROTOCOL, (efi_guid_t*)&graphics_output_guid, 0,
                                     &count, &handles) == EFI_SUCCESS &&
      handles) {
    // Console splitters can retain stale mode descriptions. Prefer a hardware
    // GOP, first matching the console, then an active display with EDID.
    for (unsigned pass = 0; pass < 3 && !selected; ++pass) {
      for (uint64_t i = 0; i < count && !selected; ++i) {
        efi_graphics_output_t* graphics = 0;
        void* path = 0;
        if (services->handle_protocol(handles[i], (efi_guid_t*)&device_path_guid, &path) !=
                EFI_SUCCESS ||
            !path ||
            services->handle_protocol(handles[i], (efi_guid_t*)&graphics_output_guid,
                                      (void**)&graphics) != EFI_SUCCESS ||
            !graphics)
          continue;
        efi_edid_active_t* edid = framebuffer_edid(handles[i]);
        uint32_t width, height;
        const int associated =
            handles[i] == g_system_table->console_out_handle ||
            (console_address && graphics->mode && graphics->mode->framebuffer == console_address);
        unsigned priority = 2;
        if (associated)
          priority = 0;
        else if (preferred_framebuffer_size(edid, &width, &height))
          priority = 1;
        if (priority == pass)
          selected = select_framebuffer_mode(graphics, services->free_pool, edid, &framebuffer);
      }
    }
  }
  if (handles)
    services->free_pool(handles);
  if (!selected &&
      !select_framebuffer_mode(console, services->free_pool,
                               framebuffer_edid(g_system_table->console_out_handle), &framebuffer))
    return;
  // No console output or mode changes may follow this handoff before firmware exit.
  info->framebuffer = framebuffer.address;
  info->framebuffer_width = framebuffer.width;
  info->framebuffer_height = framebuffer.height;
  info->framebuffer_pitch = framebuffer.pitch;
  info->framebuffer_bpp = framebuffer.bpp;
  info->framebuffer_format = framebuffer.format;
  info->flags |= BOOTSTRAP_FLAG_FRAMEBUFFER;
}
 
static void zero(void* p, uint64_t size) {
  uint8_t* bytes = (uint8_t*)p;
  while (size--)
    *bytes++ = 0;
}
 
static void copy(void* destination, const void* source, uint64_t size) {
  uint8_t* d = (uint8_t*)destination;
  const uint8_t* s = (const uint8_t*)source;
  while (size--)
    *d++ = *s++;
}
 
static int build_artifact_path(efi_loaded_image_t* loaded_image, const uefi_load_options_t* options,
                               const efi_char16_t* name, efi_char16_t* path,
                               uint64_t path_capacity) {
  const efi_char16_t* prefix = current_prefix;
  uint64_t prefix_length = sizeof(current_prefix) / sizeof(current_prefix[0]) - 1;
  efi_char16_t* file_path = 0;
  uint64_t file_path_length = 0;
  efi_device_path_t* node = loaded_image ? (efi_device_path_t*)loaded_image->file_path : 0;
 
  while (node && node->length >= sizeof(*node) && node->type != EFI_DEVICE_PATH_TYPE_END) {
    if (node->type == EFI_DEVICE_PATH_TYPE_MEDIA &&
        node->subtype == EFI_DEVICE_PATH_SUBTYPE_FILE_PATH) {
      file_path = (efi_char16_t*)(node + 1);
      file_path_length = (node->length - sizeof(*node)) / sizeof(efi_char16_t);
    }
    node = (efi_device_path_t*)((uint8_t*)node + node->length);
  }
 
  uint64_t length = 0;
  uint64_t separator = 0;
  int has_separator = 0;
  if (file_path) {
    while (length < file_path_length && file_path[length]) {
      if (file_path[length] == '\\') {
        separator = length;
        has_separator = 1;
      }
      ++length;
    }
  }
 
  if (file_path && has_separator) {
    prefix = file_path;
    prefix_length = separator + 1;
    uint64_t removable_media_length = sizeof(removable_prefix) / sizeof(removable_prefix[0]) - 1;
    if (prefix_length == removable_media_length) {
      int is_removable_media_path = 1;
      for (uint64_t i = 0; i < removable_media_length; ++i) {
        if (prefix[i] != removable_prefix[i])
          is_removable_media_path = 0;
      }
      if (is_removable_media_path) {
        prefix = current_prefix;
        prefix_length = sizeof(current_prefix) / sizeof(current_prefix[0]) - 1;
      }
    }
  }
 
  if (options->directory == UEFI_BOOT_DIRECTORY_KNOWN_GOOD) {
    prefix = known_good_prefix;
    prefix_length = sizeof(known_good_prefix) / sizeof(known_good_prefix[0]) - 1;
  } else if (options->directory == UEFI_BOOT_DIRECTORY_CURRENT) {
    prefix = current_prefix;
    prefix_length = sizeof(current_prefix) / sizeof(current_prefix[0]) - 1;
  } else if (options->directory == UEFI_BOOT_DIRECTORY_DEBUG) {
    prefix = debug_prefix;
    prefix_length = sizeof(debug_prefix) / sizeof(debug_prefix[0]) - 1;
  }
 
  uint64_t name_length = 0;
  while (name[name_length])
    ++name_length;
  if (prefix_length + name_length + 1 > path_capacity)
    return 0;
 
  for (uint64_t i = 0; i < prefix_length; ++i)
    path[i] = prefix[i];
  for (uint64_t i = 0; i < name_length; ++i)
    path[prefix_length + i] = name[i];
  path[prefix_length + name_length] = 0;
  return 1;
}
 
static void print(const efi_char16_t* message) {
  if (g_system_table && g_system_table->con_out)
    g_system_table->con_out->output_string(g_system_table->con_out, (efi_char16_t*)message);
}
 
static uint64_t pages_for(uint64_t size) {
  return (size + EFI_PAGE_SIZE - 1) >> 12;
}
 
static void* allocate_pages(uint64_t size, uint64_t max_address) {
  uint64_t address = max_address;
  efi_status_t status = ((efi_allocate_pages_t)g_system_table->boot_services->allocate_pages)(
      EFI_ALLOCATE_MAX_ADDRESS, EFI_LOADER_DATA, pages_for(size), &address);
  return status == EFI_SUCCESS ? (void*)address : 0;
}
 
static void* allocate_at(uint64_t address, uint64_t size) {
  uint64_t location = address;
  efi_status_t status = ((efi_allocate_pages_t)g_system_table->boot_services->allocate_pages)(
      EFI_ALLOCATE_ADDRESS, EFI_LOADER_DATA, pages_for(size), &location);
  return status == EFI_SUCCESS ? (void*)location : 0;
}
 
static uint64_t file_size(efi_file_t* file) {
  uint8_t info[256];
  uint64_t size = sizeof(info);
  if (file->get_info(file, (efi_guid_t*)&file_info_guid, &size, info) != EFI_SUCCESS)
    return 0;
  return *(uint64_t*)(info + 8);
}
 
static void* read_file(efi_file_t* root, efi_char16_t* path, uint64_t* length) {
  efi_file_t* file = 0;
  if (root->open(root, &file, path, EFI_FILE_MODE_READ, 0) != EFI_SUCCESS)
    return 0;
  uint64_t size = file_size(file);
  void* buffer = size ? allocate_pages(size + 1, 0xffffffffULL) : 0;
  if (!buffer || file->read(file, &size, buffer) != EFI_SUCCESS) {
    file->close(file);
    return 0;
  }
  ((uint8_t*)buffer)[size] = 0;
  file->close(file);
  *length = size;
  return buffer;
}
 
static int open_boot_filesystem(efi_loaded_image_t* loaded_image, efi_char16_t* probe_path,
                                efi_simple_file_system_t** filesystem_out, efi_file_t** root_out) {
  efi_handle_t filesystem_handle = loaded_image->device_handle;
  efi_simple_file_system_t* filesystem = 0;
  efi_file_t* root = 0;
  efi_file_t* probe = 0;
 
  if (g_system_table->boot_services->handle_protocol(filesystem_handle,
                                                     (efi_guid_t*)&simple_file_system_guid,
                                                     (void**)&filesystem) == EFI_SUCCESS &&
      filesystem->open_volume(filesystem, &root) == EFI_SUCCESS &&
      root->open(root, &probe, probe_path, EFI_FILE_MODE_READ, 0) == EFI_SUCCESS) {
    probe->close(probe);
    *filesystem_out = filesystem;
    *root_out = root;
    return 1;
  }
  if (root)
    root->close(root);
 
  void* device_path = loaded_image->file_path;
  if (g_system_table->boot_services->locate_device_path &&
      g_system_table->boot_services->locate_device_path(
          (efi_guid_t*)&simple_file_system_guid, &device_path, &filesystem_handle) == EFI_SUCCESS &&
      g_system_table->boot_services->handle_protocol(filesystem_handle,
                                                     (efi_guid_t*)&simple_file_system_guid,
                                                     (void**)&filesystem) == EFI_SUCCESS &&
      filesystem->open_volume(filesystem, &root) == EFI_SUCCESS &&
      root->open(root, &probe, probe_path, EFI_FILE_MODE_READ, 0) == EFI_SUCCESS) {
    probe->close(probe);
    *filesystem_out = filesystem;
    *root_out = root;
    return 1;
  }
  if (root)
    root->close(root);
 
  // GRUB can chainload an EFI image without giving it the ESP's filesystem handle.
  if (!g_system_table->boot_services->locate_handle_buffer)
    return 0;
  efi_handle_t* handles = 0;
  uint64_t handle_count = 0;
  if (g_system_table->boot_services->locate_handle_buffer(EFI_LOCATE_BY_PROTOCOL,
                                                          (efi_guid_t*)&simple_file_system_guid, 0,
                                                          &handle_count, &handles) != EFI_SUCCESS)
    return 0;
  for (uint64_t i = 0; i < handle_count; ++i) {
    filesystem = 0;
    root = 0;
    if (g_system_table->boot_services->handle_protocol(handles[i],
                                                       (efi_guid_t*)&simple_file_system_guid,
                                                       (void**)&filesystem) != EFI_SUCCESS ||
        filesystem->open_volume(filesystem, &root) != EFI_SUCCESS)
      continue;
    if (root->open(root, &probe, probe_path, EFI_FILE_MODE_READ, 0) == EFI_SUCCESS) {
      probe->close(probe);
      *filesystem_out = filesystem;
      *root_out = root;
      return 1;
    }
    root->close(root);
  }
  return 0;
}
 
static void enter_kernel(uint64_t entry, uint64_t info) __attribute__((noreturn));
static void enter_kernel(uint64_t entry, uint64_t info) {
  __asm__ volatile("cli\n\tmov %0, %%rbx\n\tjmp *%1" : : "r"(info), "r"(entry) : "memory");
  for (;;)
    __asm__ volatile("hlt");
}
 
efi_status_t efi_main(efi_handle_t image, efi_system_table_t* system_table) {
  g_system_table = system_table;
  print((efi_char16_t*)L"Pedigree UEFI loader\r\n");
 
  efi_loaded_image_t* loaded_image = 0;
  if (system_table->boot_services->handle_protocol(image, (efi_guid_t*)&loaded_image_guid,
                                                   (void**)&loaded_image) != EFI_SUCCESS) {
    print((efi_char16_t*)L"UEFI: loaded-image protocol failed\r\n");
    return 1;
  }
 
  uefi_load_options_t options;
  if (!parse_load_options(loaded_image->load_options, loaded_image->load_options_size, &options)) {
    print((efi_char16_t*)L"UEFI: invalid load options\r\n");
    return 1;
  }
 
  static const efi_char16_t kernel_name[] = L"kernel";
  static const efi_char16_t initrd_name[] = L"initrd.tar";
  static const efi_char16_t cmdline_name[] = L"cmdline";
  efi_char16_t kernel_path[256];
  efi_char16_t initrd_path[256];
  efi_char16_t cmdline_path[256];
  if (!build_artifact_path(loaded_image, &options, kernel_name, kernel_path, 256) ||
      !build_artifact_path(loaded_image, &options, initrd_name, initrd_path, 256) ||
      !build_artifact_path(loaded_image, &options, cmdline_name, cmdline_path, 256)) {
    print((efi_char16_t*)L"UEFI: loader path is too long\r\n");
    return 1;
  }
  efi_simple_file_system_t* filesystem = 0;
  efi_file_t* root = 0;
  if (!open_boot_filesystem(loaded_image, kernel_path, &filesystem, &root)) {
    print((efi_char16_t*)L"UEFI: filesystem protocol failed\r\n");
    return 1;
  }
  uint64_t kernel_length = 0;
  uint64_t initrd_length = 0;
  uint64_t cmdline_length = 0;
  uint8_t* kernel_file = (uint8_t*)read_file(root, kernel_path, &kernel_length);
  uint8_t* initrd = (uint8_t*)read_file(root, initrd_path, &initrd_length);
  uint8_t* cmdline = (uint8_t*)read_file(root, cmdline_path, &cmdline_length);
  if (!kernel_file || !initrd || !cmdline) {
    print((efi_char16_t*)L"UEFI: file read failed\r\n");
    return 1;
  }
  if (options.arguments_length) {
    char* combined = (char*)allocate_pages(UEFI_COMMAND_LINE_CAPACITY, 0xffffffffULL);
    if (!combined || !append_load_options(cmdline, cmdline_length, &options, combined,
                                          UEFI_COMMAND_LINE_CAPACITY)) {
      print((efi_char16_t*)L"UEFI: invalid combined command line\r\n");
      return 1;
    }
    cmdline = (uint8_t*)combined;
  }
  elf64_header_t* header = (elf64_header_t*)kernel_file;
  if (kernel_length < sizeof(*header) || header->ident[0] != 0x7f || header->ident[1] != 'E' ||
      header->ident[2] != 'L' || header->ident[3] != 'F' || header->ident[4] != 2 ||
      header->machine != 0x3e) {
    print((efi_char16_t*)L"UEFI: invalid ELF\r\n");
    return 1;
  }
  print((efi_char16_t*)L"UEFI: loading kernel\r\n");
 
  for (uint16_t i = 0; i < header->program_header_count; ++i) {
    elf64_program_header_t* program =
        (elf64_program_header_t*)(kernel_file + header->program_header_offset +
                                  i * header->program_header_size);
    if (program->type != PT_LOAD)
      continue;
    uint8_t* destination = (uint8_t*)allocate_at(program->physical_address, program->memory_size);
    if (!destination) {
      print((efi_char16_t*)L"UEFI: kernel allocation failed\r\n");
      return 1;
    }
    copy(destination, kernel_file + program->offset, program->file_size);
    if (program->memory_size > program->file_size)
      zero(destination + program->file_size, program->memory_size - program->file_size);
  }
 
  if (header->section_header_offset && header->section_header_count) {
    for (uint16_t i = 0; i < header->section_header_count; ++i) {
      elf64_section_header_t* section =
          (elf64_section_header_t*)(kernel_file + header->section_header_offset +
                                    i * header->section_header_size);
      if (!(section->flags & SHF_ALLOC) && section->size)
        section->address = PHYS_ALIAS + (uint64_t)kernel_file + section->offset;
    }
  }
 
  bootstrap_info_t* info = (bootstrap_info_t*)allocate_pages(4 * EFI_PAGE_SIZE, 0xffffffffULL);
  void* memory_map = allocate_pages(128 * EFI_PAGE_SIZE, 0xffffffffULL);
  void* normalized_memory_map = allocate_pages(128 * EFI_PAGE_SIZE, 0xffffffffULL);
  if (!info || !memory_map || !normalized_memory_map) {
    print((efi_char16_t*)L"UEFI: metadata allocation failed\r\n");
    return 1;
  }
  bootstrap_module_t* modules = (bootstrap_module_t*)((uint8_t*)info + EFI_PAGE_SIZE);
  zero(info, 4 * EFI_PAGE_SIZE);
  modules[0].base = (uint64_t)initrd;
  modules[0].end = modules[0].base + initrd_length;
  info->flags =
      BOOTSTRAP_FLAG_CMDLINE | BOOTSTRAP_FLAG_MODULES | BOOTSTRAP_FLAG_ELF | BOOTSTRAP_FLAG_UEFI;
  info->module_count = 1;
  info->modules = PHYS_ALIAS + (uint64_t)modules;
  info->section_count = header->section_header_count;
  info->section_entry_size = header->section_header_size;
  info->section_string_index = header->section_string_index;
  info->section_headers = PHYS_ALIAS + (uint64_t)kernel_file + header->section_header_offset;
  info->command_line = PHYS_ALIAS + (uint64_t)cmdline;
 
  for (uint64_t i = 0; i < system_table->configuration_table_entries; ++i) {
    efi_configuration_table_t* table =
        (efi_configuration_table_t*)system_table->configuration_table + i;
    if (table->vendor_guid.a == acpi20_guid.a && table->vendor_guid.b == acpi20_guid.b &&
        table->vendor_guid.c == acpi20_guid.c) {
      info->flags |= BOOTSTRAP_FLAG_ACPI;
      info->acpi_rsdp = PHYS_ALIAS + (uint64_t)table->vendor_table;
    }
    if (table->vendor_guid.a == smbios_guid.a && table->vendor_guid.b == smbios_guid.b &&
        table->vendor_guid.c == smbios_guid.c) {
      info->flags |= BOOTSTRAP_FLAG_SMBIOS;
      info->smbios = PHYS_ALIAS + (uint64_t)table->vendor_table;
    }
  }
 
  prepare_framebuffer(info);
 
  uint32_t normalized_bytes = 0;
  int exit_attempted = 0;
  const efi_get_memory_map_t get_map =
      (efi_get_memory_map_t)system_table->boot_services->get_memory_map;
  const efi_exit_boot_services_t exit_services = system_table->boot_services->exit_boot_services;
  const efi_status_t exit_status =
      exit_boot_services_with_map(get_map, exit_services, image, memory_map, 128 * EFI_PAGE_SIZE,
                                  (bootstrap_memory_map_entry_t*)normalized_memory_map,
                                  128 * EFI_PAGE_SIZE, &normalized_bytes, &exit_attempted);
  if (exit_status != EFI_SUCCESS) {
    if (!exit_attempted) {
      print((efi_char16_t*)L"UEFI: invalid or unavailable memory map\r\n");
      return exit_status;
    }
    // Returning or using console protocols is unsafe after a partial firmware exit.
    __asm__ volatile("cli" ::: "memory");
    for (;;)
      __asm__ volatile("hlt");
  }
  info->flags |= BOOTSTRAP_FLAG_MEMORY_MAP;
  info->memory_map = PHYS_ALIAS + (uint64_t)normalized_memory_map;
  info->memory_map_length = normalized_bytes;
  info->memory_map_entry_size = sizeof(bootstrap_memory_map_entry_t);
 
  enter_kernel(header->entry - KERNEL_BASE, PHYS_ALIAS + (uint64_t)info);
}