vm: implement demand-paging via userspace services with vm-controller

interrupts will need to be enable to allow for requesting missing pages from userspace
services.

CMakeLists.txt

@@ -12,12 +12,11 @@ set(kernel_arch x86_64)
 set(kernel_name "Mango")
 set(kernel_exe_name "mango_kernel")
 
-set(generic_src_dirs ds init kernel libc sched util vm)
+set(generic_src_dirs ds init kernel libc sched util vm syscall)
 set(kernel_sources "")
 set(kernel_headers "")
 
 foreach (dir ${generic_src_dirs})
-	message(STATUS ${dir})
 	file(GLOB_RECURSE dir_sources ${dir}/*.c)
 	file(GLOB_RECURSE dir_headers ${dir}/*.h)
 
@@ -41,11 +40,19 @@ add_executable(${kernel_exe_name}
 target_include_directories(${kernel_exe_name} PRIVATE
 	include
 	libc/include
+	libmango/include
 	arch/${kernel_arch}/include)
 target_compile_options(${kernel_exe_name} PRIVATE
 	-nostdlib -ffreestanding
 	-Wall -Werror -pedantic -Wno-language-extension-token -Wno-unused-function  -Wno-gnu-statement-expression
-	-O2 -g -fPIC -Iinclude -Iarch/${kernel_arch}/include -Ilibc/include)
+	-g -fPIC -Iinclude -Iarch/${kernel_arch}/include -Ilibc/include)
+
+add_custom_command(
+	TARGET ${kernel_exe_name}
+	POST_BUILD
+	COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:${kernel_exe_name}> $<TARGET_FILE:${kernel_exe_name}>.debug
+	COMMAND ${CMAKE_STRIP} -g $<TARGET_FILE:${kernel_exe_name}>)
+
 target_link_libraries(${kernel_exe_name} -nostdlib -ffreestanding -lgcc)
 target_compile_definitions(${kernel_exe_name} PRIVATE BUILD_ID="0")
 

arch/user/cpu.c

@@ -1,5 +1,5 @@
 #include <unistd.h>
-#include <mango/machine/cpu.h>
+#include <kernel/machine/cpu.h>
 
 int ml_init_bootcpu(void)
 {

arch/user/hwlock.c

@@ -1,5 +1,5 @@
-#include <mango/machine/hwlock.h>
+#include <kernel/compiler.h>
-#include <mango/compiler.h>
+#include <kernel/machine/hwlock.h>
 
 void ml_hwlock_lock(ml_hwlock_t *lck)
 {

arch/user/include/mango/machine/init.h

@@ -1,6 +1,7 @@
 #ifndef MANGO_X86_64_INIT_H_
 #define MANGO_X86_64_INIT_H_
 
+#include <stddef.h>
 #include <stdint.h>
 
 #ifdef __cplusplus
@@ -8,16 +9,18 @@ extern "C" {
 #endif
 
 #define __X2(x) #x
-#define __X(x) __X2(x)
+#define __X(x)  __X2(x)
 
 #ifdef __APPLE__
-#define __define_initcall(fn, id) \
+#define __define_initcall(fn, id)                                              \
-	static initcall_t __initcall_##fn##id __used \
+	static initcall_t __initcall_##fn##id __used __section(                \
-	__section("__DATA,__initcall" __X(id) ".init") = (fn)
+		"__DATA,__initcall" __X(id) ".init")                           \
+		= (fn)
 #else
-#define __define_initcall(fn, id) \
+#define __define_initcall(fn, id)                                              \
-	static initcall_t __initcall_##fn##id __used \
+	static initcall_t __initcall_##fn##id __used __section(                \
-	__section("initcall" __X(id) "_init") = (fn)
+		"initcall" __X(id) "_init")                                    \
+		= (fn)
 #endif
 
 extern int ml_init(uintptr_t arg);

arch/user/include/mango/machine/pmap.h

@@ -1,6 +1,8 @@
 #ifndef MANGO_USER_PMAP_H_
 #define MANGO_USER_PMAP_H_
 
+#include <stdint.h>
+
 typedef uintptr_t ml_pmap_t;
 typedef uint64_t ml_pfn_t;
 

arch/user/init.c

@@ -1,11 +1,11 @@
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <mango/init.h>
+#include <kernel/init.h>
-#include <mango/memblock.h>
+#include <kernel/memblock.h>
-#include <mango/vm.h>
+#include <kernel/vm.h>
-#include <mango/object.h>
+#include <kernel/object.h>
-#include <mango/printk.h>
+#include <kernel/printk.h>
 #include <arch/stdcon.h>
 #include <sys/mman.h>
 

arch/user/initcall.c

@@ -1,4 +1,4 @@
-#include <mango/init.h>
+#include <kernel/init.h>
 
 #ifdef __APPLE__
 extern char __start_initcall0[] __asm("section$start$__DATA$__initcall0.init");

arch/user/stdcon.c

@@ -1,10 +1,10 @@
-#include <mango/libc/string.h>
+#include <kernel/libc/string.h>
-#include <mango/libc/ctype.h>
+#include <kernel/libc/ctype.h>
 #include <stdint.h>
 #include <stdio.h>
-#include <mango/console.h>
+#include <kernel/console.h>
-#include <mango/vm.h>
+#include <kernel/vm.h>
-#include <mango/printk.h>
+#include <kernel/printk.h>
 
 static void stdcon_write(struct console *con, const char *s, unsigned int len)
 {

arch/x86_64/asm-offset.c

@@ -1,10 +1,11 @@
-#include <mango/sched.h>
+#include <kernel/compiler.h>
-#include <mango/compiler.h>
+#include <kernel/sched.h>
+#include <kernel/thread.h>
 
-//size_t THREAD_sp = offsetof(struct thread, tr_sp);
+// size_t THREAD_sp = offsetof(struct thread, tr_sp);
 
-/* Use %a0 instead of %0 to prevent gcc from emitting a $ before the symbol value
+/* Use %a0 instead of %0 to prevent gcc from emitting a $ before the symbol
-   in the generated assembly.
+   value in the generated assembly.
 
    emitting
      .set TASK_sp, $56
@@ -15,11 +16,14 @@
    causes garbage symbols to be generated.
 */
 
-#define DEFINE(sym, val) \
+#define DEFINE(sym, val)                                                       \
-	asm volatile("\n.global " #sym "\n.type " #sym ", @object" "\n.set " #sym ", %a0" : : "i" (val))
+	asm volatile("\n.global " #sym "\n.type " #sym                         \
+	             ", @object"                                               \
+	             "\n.set " #sym ", %a0"                                    \
+	             :                                                         \
+	             : "i"(val))
 
-#define OFFSET(sym, str, mem) \
+#define OFFSET(sym, str, mem) DEFINE(sym, offsetof(str, mem))
-	DEFINE(sym, offsetof(str, mem))
 
 static void __used common(void)
 {

arch/x86_64/config.cmake

@@ -2,9 +2,3 @@ target_compile_options(${kernel_exe_name} PRIVATE
 	-z max-page-size=0x1000 -m64 -mcmodel=large -mno-red-zone -mno-mmx
 	-mno-sse -mno-sse2 -D_64BIT -DBYTE_ORDER=1234)
 target_link_libraries(${kernel_exe_name} "-z max-page-size=0x1000" "-T ${CMAKE_CURRENT_SOURCE_DIR}/arch/x86_64/layout.ld")
-
-add_custom_command(TARGET ${kernel_exe_name} POST_BUILD
-	COMMAND ${BUILD_TOOLS_DIR}/e64patch $<TARGET_FILE:${kernel_exe_name}>
-	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-	COMMENT "Patching kernel elf64 image"
-)

arch/x86_64/cpu.c

@@ -1,11 +1,14 @@
-#include <mango/machine/cpu.h>
 #include <arch/msr.h>
+#include <kernel/machine/cpu.h>
 
 int ml_cpu_block_init(ml_cpu_block *p)
 {
 	p->c_this = p;
 	gdt_init(&p->c_gdt, &p->c_gdt_ptr);
 	idt_init(&p->c_idt_ptr);
+	tss_init(&p->c_tss, &p->c_tss_ptr);
+
+	gdt_write_tss(&p->c_gdt, &p->c_tss);
 	return 0;
 }
 
@@ -13,6 +16,27 @@ int ml_cpu_block_use(ml_cpu_block *p)
 {
 	gdt_load(&p->c_gdt_ptr);
 	idt_load(&p->c_idt_ptr);
+	tss_load(&p->c_tss);
 	wrmsr(MSR_GS_BASE, (uint64_t)p);
 	return 0;
 }
+
+virt_addr_t ml_cpu_block_get_kstack(ml_cpu_block *p)
+{
+	return tss_get_kstack(&p->c_tss);
+}
+
+virt_addr_t ml_cpu_block_get_ustack(ml_cpu_block *p)
+{
+	return tss_get_ustack(&p->c_tss);
+}
+
+void ml_cpu_block_set_kstack(ml_cpu_block *p, virt_addr_t sp)
+{
+	tss_set_kstack(&p->c_tss, sp);
+}
+
+void ml_cpu_block_set_ustack(ml_cpu_block *p, virt_addr_t sp)
+{
+	tss_set_ustack(&p->c_tss, sp);
+}

arch/x86_64/e820.c

@@ -1,8 +1,8 @@
-#include "mango/types.h"
-#include <mango/memblock.h>
-#include <mango/printk.h>
-#include <mango/util.h>
 #include <arch/e820.h>
+#include <kernel/memblock.h>
+#include <kernel/printk.h>
+#include <kernel/types.h>
+#include <kernel/util.h>
 
 void e820_scan(multiboot_memory_map_t *mmap, size_t len)
 {
@@ -36,7 +36,9 @@ void e820_scan(multiboot_memory_map_t *mmap, size_t len)
 		}
 
 		printk("e820: [mem 0x%016llx-0x%016llx] %s",
-		       entry->addr, entry->addr + entry->len - 1, type);
+		       entry->addr,
+		       entry->addr + entry->len - 1,
+		       type);
 
 		memblock_add(entry->addr, entry->len);
 
@@ -53,7 +55,12 @@ void e820_scan(multiboot_memory_map_t *mmap, size_t len)
 
 	char str_mem_total[64], str_mem_reserved[64];
 	data_size_to_string(mem_total, str_mem_total, sizeof str_mem_total);
-	data_size_to_string(mem_reserved, str_mem_reserved, sizeof str_mem_reserved);
+	data_size_to_string(
+		mem_reserved,
+		str_mem_reserved,
+		sizeof str_mem_reserved);
 
-	printk("e820: total memory: %s, hw reserved: %s", str_mem_total, str_mem_reserved);
+	printk("e820: total memory: %s, hw reserved: %s",
+	       str_mem_total,
+	       str_mem_reserved);
 }

arch/x86_64/gdt.c

@@ -1,5 +1,8 @@
-#include <mango/libc/string.h>
 #include <arch/gdt.h>
+#include <arch/tss.h>
+#include <kernel/libc/string.h>
+#include <kernel/types.h>
+#include <stddef.h>
 
 static void init_entry(struct gdt_entry *entry, int access, int flags)
 {
@@ -15,13 +18,42 @@ int gdt_init(struct gdt *gdt, struct gdt_ptr *gdtp)
 {
 	memset(&gdt->g_entries[0], 0x0, sizeof gdt->g_entries[0]);
 
-	init_entry(&gdt->g_entries[1], GDT_A_PRESENT | GDT_A_CODEREAD | GDT_A_CODE, GDT_F_64BIT);
+	init_entry(
-	init_entry(&gdt->g_entries[2], GDT_A_PRESENT | GDT_A_DATAWRITE | GDT_A_DATA, GDT_F_64BIT);
+		&gdt->g_entries[1],
-	init_entry(&gdt->g_entries[3], GDT_A_PRESENT | GDT_A_USER | GDT_A_CODEREAD | GDT_A_CODE, GDT_F_64BIT);
+		GDT_A_PRESENT | GDT_A_CODEREAD | GDT_A_CODE,
-	init_entry(&gdt->g_entries[4], GDT_A_PRESENT | GDT_A_USER | GDT_A_DATAWRITE | GDT_A_DATA, GDT_F_64BIT);
+		GDT_F_64BIT);
+	init_entry(
+		&gdt->g_entries[2],
+		GDT_A_PRESENT | GDT_A_DATAWRITE | GDT_A_DATA,
+		GDT_F_64BIT);
+	init_entry(
+		&gdt->g_entries[3],
+		GDT_A_PRESENT | GDT_A_USER | GDT_A_DATAWRITE | GDT_A_DATA,
+		GDT_F_64BIT);
+	init_entry(
+		&gdt->g_entries[4],
+		GDT_A_PRESENT | GDT_A_USER | GDT_A_CODEREAD | GDT_A_CODE,
+		GDT_F_64BIT);
 
 	gdtp->g_ptr = (uint64_t)gdt;
 	gdtp->g_limit = sizeof(*gdt) - 1;
 
 	return 0;
 }
+
+void gdt_write_tss(struct gdt *gdt, struct tss *tss)
+{
+	struct tss_gdt_entry *tss_entry = &gdt->g_tss;
+
+	virt_addr_t base = (virt_addr_t)tss;
+	size_t limit = sizeof *tss;
+
+	tss_entry->ge_base_low = (base & 0xFFFF);
+	tss_entry->ge_base_mid = (base >> 16) & 0xFF;
+	tss_entry->ge_base_hi = (base >> 24) & 0xFF;
+	tss_entry->ge_base_ext = (base >> 32) & 0xFFFFFFFF;
+	tss_entry->ge_limit_low = (limit & 0xFFFF);
+	tss_entry->ge_gran = (limit >> 16) & 0xF;
+	tss_entry->ge_access = 0xE9;
+	tss_entry->ge_reserved = 0;
+}

arch/x86_64/hwlock.S

@@ -11,6 +11,41 @@ ml_hwlock_lock:
 
 	mov $1, %ecx
 
+	mfence
+
+1:	mov $0, %eax
+	lock cmpxchg %ecx, (%rdi)
+	jne 1b
+
+	pop %rbp
+	ret
+
+
+	.global ml_hwlock_unlock
+	.type ml_hwlock_unlock, @function
+
+	/* %rdi = pointer to ml_hwlock_t (int) */
+ml_hwlock_unlock:
+	push %rbp
+	mov %rsp, %rbp
+
+	movl $0, (%rdi)
+	mfence
+
+	pop %rbp
+	ret
+
+
+	.global ml_hwlock_lock_irq
+	.type ml_hwlock_lock_irq, @function
+
+	/* %rdi = pointer to ml_hwlock_t (int) */
+ml_hwlock_lock_irq:
+	push %rbp
+	mov %rsp, %rbp
+
+	mov $1, %ecx
+
 	cli
 	mfence
 
@@ -21,11 +56,12 @@ ml_hwlock_lock:
 	pop %rbp
 	ret
 
-	.global ml_hwlock_unlock
+
-	.type ml_hwlock_unlock, @function
+	.global ml_hwlock_unlock_irq
+	.type ml_hwlock_unlock_irq, @function
 
 	/* %rdi = pointer to ml_hwlock_t (int) */
-ml_hwlock_unlock:
+ml_hwlock_unlock_irq:
 	push %rbp
 	mov %rsp, %rbp
 
@@ -42,7 +78,7 @@ ml_hwlock_unlock:
 
 	/* %rdi = pointer to ml_hwlock_t (int)
 	   %rsi = pointer to quadword to store rflags in */
-ml_hwlock_lock_irqsave:	
+ml_hwlock_lock_irqsave:
 	push %rbp
 	mov %rsp, %rbp
 
@@ -62,6 +98,7 @@ ml_hwlock_lock_irqsave:
 	pop %rbp
 	ret
 
+
 	.global ml_hwlock_unlock_irqrestore
 	.type ml_hwlock_unlock_irqrestore, @function
 

arch/x86_64/include/arch/gdt.h

@@ -1,26 +1,27 @@
 #ifndef ARCH_GDT_H_
 #define ARCH_GDT_H_
 
-#include <mango/compiler.h>
+#include <arch/tss.h>
+#include <kernel/compiler.h>
 #include <stdint.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#define NR_GDT_ENTRIES 5
+#define NR_GDT_ENTRIES  5
 
-#define GDT_A_PRESENT		(1 << 7)
+#define GDT_A_PRESENT   (1 << 7)
-#define GDT_A_USER		(3 << 5)
+#define GDT_A_USER      (3 << 5)
-#define GDT_A_CODE		(3 << 3)
+#define GDT_A_CODE      (3 << 3)
-#define GDT_A_DATA		(8 << 1)
+#define GDT_A_DATA      (8 << 1)
-#define GDT_A_RPLK		(1 << 5)
+#define GDT_A_RPLK      (1 << 5)
-#define GDT_A_CODEREAD		(1 << 1)
+#define GDT_A_CODEREAD  (1 << 1)
-#define GDT_A_DATAWRITE		(1 << 1)
+#define GDT_A_DATAWRITE (1 << 1)
-#define GDT_A_GROWUP		(1 << 5)
+#define GDT_A_GROWUP    (1 << 5)
-#define GDT_F_64BIT		(0x2F)
+#define GDT_F_64BIT     (0x2F)
-#define GDT_F_32BIT		(0x4F)
+#define GDT_F_32BIT     (0x4F)
-#define GDT_F_16BIT		(0xF)
+#define GDT_F_16BIT     (0xF)
 
 struct gdt_entry {
 	uint16_t ge_limit_low;
@@ -33,6 +34,7 @@ struct gdt_entry {
 
 struct gdt {
 	struct gdt_entry g_entries[NR_GDT_ENTRIES];
+	struct tss_gdt_entry g_tss;
 } __packed;
 
 struct gdt_ptr {
@@ -41,6 +43,7 @@ struct gdt_ptr {
 } __packed;
 
 extern int gdt_init(struct gdt *gdt, struct gdt_ptr *gdtp);
+extern void gdt_write_tss(struct gdt *gdt, struct tss *tss);
 extern int gdt_load(struct gdt_ptr *gdtp);
 
 #ifdef __cplusplus

arch/x86_64/include/arch/irq.h

@@ -1,8 +1,8 @@
 #ifndef ARCH_IRQ_H_
 #define ARCH_IRQ_H_
 
-#include <mango/compiler.h>
+#include <kernel/compiler.h>
-#include <mango/queue.h>
+#include <kernel/queue.h>
 #include <stdint.h>
 
 #ifdef __cplusplus
@@ -11,6 +11,8 @@ extern "C" {
 
 #define NR_IDT_ENTRIES 256
 
+struct ml_cpu_context;
+
 enum irq_vector {
 	IRQ0 = 32,
 	IRQ1,
@@ -35,13 +37,6 @@ struct irq_hook {
 	int (*irq_callback)(void);
 };
 
-struct cpu_context {
-	uint64_t r15, r14, r13, r12, r11, r10, r9, r8;
-	uint64_t rdi, rsi, rbp, unused_rsp, rbx, rdx, rcx, rax;
-	uint64_t int_no, err_no;
-	uint64_t rip, cs, rflags, rsp, ss;
-} __packed;
-
 struct idt_entry {
 	uint16_t base_low;
 	uint16_t selector;
@@ -64,7 +59,7 @@ struct idt_ptr {
 	uintptr_t i_base;
 } __packed;
 
-typedef void (*int_hook)(struct cpu_context *);
+typedef void (*int_hook)(struct ml_cpu_context *);
 
 extern int idt_init(struct idt_ptr *idtp);
 extern int idt_load(struct idt_ptr *idtp);

arch/x86_64/include/arch/msr.h

@@ -3,7 +3,8 @@
 
 #include <stdint.h>
 
-#define MSR_GS_BASE 0xC0000101
+#define MSR_GS_BASE        0xC0000101
+#define MSR_KERNEL_GS_BASE 0xC0000102
 
 #ifdef __cplusplus
 extern "C" {

arch/x86_64/include/arch/paging.h

@@ -1,8 +1,8 @@
 #ifndef ARCH_PAGING_H_
 #define ARCH_PAGING_H_
 
-#include <mango/types.h>
+#include <kernel/types.h>
-#include <mango/compiler.h>
+#include <kernel/compiler.h>
 
 #ifdef __cplusplus
 extern "C" {

arch/x86_64/include/arch/tss.h

@@ -0,0 +1,55 @@
+#ifndef ARCH_TSS_H_
+#define ARCH_TSS_H_
+
+#include <kernel/compiler.h>
+#include <kernel/types.h>
+#include <stdint.h>
+
+#define TSS_GDT_INDEX 5
+
+struct tss {
+	uint32_t res0;
+	uint64_t rsp0;
+	uint64_t rsp1;
+	uint64_t rsp2;
+	uint64_t res1;
+	uint64_t ist1;
+	uint64_t ist2;
+	uint64_t ist3;
+	uint64_t ist4;
+	uint64_t ist5;
+	uint64_t ist6;
+	uint64_t ist7;
+	uint64_t res2;
+	uint16_t res3;
+	uint16_t iomap_offset;
+} __packed;
+
+struct tss_gdt_entry {
+	/* these fields are copied from struct gdt_entry */
+	uint16_t ge_limit_low;
+	uint16_t ge_base_low;
+	uint8_t ge_base_mid;
+	uint8_t ge_access;
+	uint8_t ge_gran;
+	uint8_t ge_base_hi;
+	/* these fields are specific to the TSS entry */
+	uint32_t ge_base_ext;
+	uint32_t ge_reserved;
+} __packed;
+
+struct tss_ptr {
+	uint16_t tss_limit;
+	uint64_t tss_base;
+} __packed;
+
+extern void tss_init(struct tss *tss, struct tss_ptr *ptr);
+extern void tss_load(struct tss *tss);
+
+extern virt_addr_t tss_get_kstack(struct tss *tss);
+extern virt_addr_t tss_get_ustack(struct tss *tss);
+
+extern void tss_set_kstack(struct tss *tss, virt_addr_t sp);
+extern void tss_set_ustack(struct tss *tss, virt_addr_t sp);
+
+#endif

arch/x86_64/include/kernel/machine/cpu.h

@@ -0,0 +1,77 @@
+#ifndef KERNEL_X86_64_CPU_H_
+#define KERNEL_X86_64_CPU_H_
+
+#include <arch/gdt.h>
+#include <arch/irq.h>
+#include <arch/tss.h>
+#include <kernel/types.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define ML_BIG_ENDIAN            0
+
+#define ml_cpu_block_get_id(p)   ((p)->c_cpu_id)
+#define ml_cpu_block_get_data(p) ((p)->c_data)
+
+#if 0
+#define ml_read_sp(sp, bp)                                                     \
+	asm volatile("mov %%rsp, %0" : "=r"(sp));                              \
+	asm volatile("mov %%rbp, %0" : "=r"(bp));
+#endif
+
+struct cpu_data;
+
+typedef struct ml_cpu_block {
+	struct ml_cpu_block *c_this;
+
+	struct gdt c_gdt;
+	struct gdt_ptr c_gdt_ptr;
+
+	struct tss c_tss;
+	struct tss_ptr c_tss_ptr;
+
+	struct idt_ptr c_idt_ptr;
+	unsigned int c_cpu_id;
+
+	struct cpu_data *c_data;
+} ml_cpu_block;
+
+struct ml_int_context {
+	uint64_t rip, cs, rflags, rsp, ss;
+};
+
+struct ml_cpu_context {
+	uint64_t r15, r14, r13, r12, r11, r10, r9, r8;
+	uint64_t rdi, rsi, rbp, unused_rsp, rbx, rdx, rcx, rax;
+	uint64_t int_no, err_no;
+	uint64_t rip, cs, rflags, rsp, ss;
+} __packed;
+
+#define ml_cpu_pause()   __asm__ __volatile__("hlt")
+#define ml_cpu_relax()   __asm__ __volatile__("pause")
+
+#define ml_int_disable() __asm__ __volatile__("cli")
+#define ml_int_enable()  __asm__ __volatile__("sti")
+
+extern int ml_init_bootcpu(void);
+
+extern int ml_cpu_block_init(ml_cpu_block *p);
+extern int ml_cpu_block_use(ml_cpu_block *p);
+
+extern virt_addr_t ml_cpu_block_get_ustack(ml_cpu_block *p);
+extern virt_addr_t ml_cpu_block_get_kstack(ml_cpu_block *p);
+
+extern void ml_cpu_block_set_ustack(ml_cpu_block *p, virt_addr_t sp);
+extern void ml_cpu_block_set_kstack(ml_cpu_block *p, virt_addr_t sp);
+
+/* defined in cpu_ctrl.S */
+extern void ml_halt_cpu(void);
+extern ml_cpu_block *ml_this_cpu(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

arch/x86_64/include/kernel/machine/hwlock.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_X86_64_HWLOCK_H_
+#ifndef KERNEL_X86_64_HWLOCK_H_
-#define MANGO_X86_64_HWLOCK_H_
+#define KERNEL_X86_64_HWLOCK_H_
 
 #define ML_HWLOCK_INIT (0)
 
@@ -12,6 +12,9 @@ typedef int ml_hwlock_t;
 extern void ml_hwlock_lock(ml_hwlock_t *lck);
 extern void ml_hwlock_unlock(ml_hwlock_t *lck);
 
+extern void ml_hwlock_lock_irq(ml_hwlock_t *lck);
+extern void ml_hwlock_unlock_irq(ml_hwlock_t *lck);
+
 extern void ml_hwlock_lock_irqsave(ml_hwlock_t *lck, unsigned long *flags);
 extern void ml_hwlock_unlock_irqrestore(ml_hwlock_t *lck, unsigned long flags);
 

arch/x86_64/include/kernel/machine/init.h

@@ -0,0 +1,28 @@
+#ifndef KERNEL_X86_64_INIT_H_
+#define KERNEL_X86_64_INIT_H_
+
+#include <stddef.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define __X2(x) #x
+#define __X(x)  __X2(x)
+
+#define __define_initcall(fn, id)                                              \
+	static initcall_t __initcall_##fn##id __used __section(                \
+		".initcall" __X(id) ".init")                                   \
+		= (fn)
+
+extern int ml_init(uintptr_t arg);
+
+extern const struct framebuffer_varinfo *bootfb_varinfo(void);
+extern const struct framebuffer_fixedinfo *bootfb_fixedinfo(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

arch/x86_64/include/kernel/machine/irq.h

@@ -0,0 +1,5 @@
+#ifndef KERNEL_X86_64_IRQ_H_
+#define KERNEL_X86_64_IRQ_H_
+
+
+#endif

arch/x86_64/include/kernel/machine/panic.h

@@ -0,0 +1,12 @@
+#ifndef KERNEL_X86_64_PANIC_H_
+#define KERNEL_X86_64_PANIC_H_
+
+#include <stdint.h>
+
+struct ml_cpu_context;
+
+extern void ml_print_cpu_state(struct ml_cpu_context *ctx);
+extern void ml_print_stack_trace(uintptr_t ip);
+extern void ml_print_stack_trace_irq(struct ml_cpu_context *ctx);
+
+#endif

arch/x86_64/include/kernel/machine/pmap.h

@@ -0,0 +1,11 @@
+#ifndef KERNEL_X86_64_PMAP_H_
+#define KERNEL_X86_64_PMAP_H_
+
+#include <arch/paging.h>
+
+#define ML_PMAP_INVALID ((uintptr_t)-1)
+
+typedef pml4t_ptr_t ml_pmap_t;
+typedef uint64_t ml_pfn_t;
+
+#endif

arch/x86_64/include/kernel/machine/random.h

@@ -0,0 +1,10 @@
+#ifndef KERNEL_X86_64_RANDOM_H_
+#define KERNEL_X86_64_RANDOM_H_
+
+#include <stdbool.h>
+#include <stdint.h>
+
+extern bool ml_hwrng_available(void);
+extern uint64_t ml_hwrng_generate(void);
+
+#endif

arch/x86_64/include/kernel/machine/thread.h

@@ -0,0 +1,31 @@
+#ifndef KERNEL_X86_64_THREAD_H_
+#define KERNEL_X86_64_THREAD_H_
+
+#include <kernel/sched.h>
+
+struct ml_cpu_context;
+
+/* switch from one thread to another. the stack of the `to` thread must have
+ * been prepared in one of two ways:
+ *  1) a previous call to ml_thread_switch where it was the `from` thread.
+ *  2) a call to ml_thread_prepare_kernel_context
+ * the switch occurs entirely with kernel mode. a further return from an
+ * interrupt context is then used to return to usermode if necessary.
+ */
+extern void ml_thread_switch(struct thread *from, struct thread *to);
+
+/* perform the initial transition to userspace. the stack must be prepared using
+ * ml_thread_prepare_user_context before this function can be used */
+extern void ml_thread_switch_user(void);
+/* prepare the stack so that ml_thread_switch can jump to the specified IP/SP */
+extern void ml_thread_prepare_kernel_context(uintptr_t ip, uintptr_t *sp);
+/* prepare the stack so that ml_thread_switch_user can jump to usermode
+ * with the specified IP/user SP */
+extern kern_status_t ml_thread_prepare_user_context(
+	virt_addr_t ip,
+	virt_addr_t user_sp,
+	virt_addr_t *kernel_sp,
+	const uintptr_t *args,
+	size_t nr_args);
+
+#endif

arch/x86_64/include/kernel/machine/vm.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_X86_64_VM_H_
+#ifndef KERNEL_X86_64_VM_H_
-#define MANGO_X86_64_VM_H_
+#define KERNEL_X86_64_VM_H_
 
 /* kernel higher-half base virtual address. */
 #define VM_KERNEL_VOFFSET 0xFFFFFFFF80000000
@@ -11,14 +11,20 @@
 #define VM_PAGEMAP_BASE   0xFFFF888000000000
 #define VM_PAGEMAP_LIMIT  0xFFFFC87FFFFFFFFF
 
-#define VM_PAGE_SIZE  0x1000
+#define VM_PAGE_SIZE      0x1000
-#define VM_PAGE_MASK  (VM_PAGE_SIZE-1)
+#define VM_PAGE_MASK      (VM_PAGE_SIZE - 1)
-#define VM_PAGE_SHIFT 12
+#define VM_PAGE_SHIFT     12
 
 #define VM_PAGE_MIN_ORDER VM_PAGE_4K
 #define VM_PAGE_MAX_ORDER VM_PAGE_8M
 
-#define VM_ZONE_MIN VM_ZONE_DMA
+#define VM_ZONE_MIN       VM_ZONE_DMA
-#define VM_ZONE_MAX VM_ZONE_NORMAL
+#define VM_ZONE_MAX       VM_ZONE_NORMAL
+
+#define VM_USER_BASE      0x0000000000100000
+#define VM_USER_LIMIT     0x00007fffffffffff
+
+#define VM_KERNEL_BASE    0XFFFF800000000000
+#define VM_KERNEL_LIMIT   0XFFFFFFFFFFFFFFFF
 
 #endif

arch/x86_64/include/mango/machine/cpu.h

@@ -1,49 +0,0 @@
-#ifndef MANGO_X86_64_CPU_H_
-#define MANGO_X86_64_CPU_H_
-
-#include <arch/gdt.h>
-#include <arch/irq.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define ML_BIG_ENDIAN 0
-
-#define ml_cpu_block_get_id(p) ((p)->c_cpu_id)
-#define ml_cpu_block_get_data(p) ((p)->c_data)
-
-struct cpu_data;
-
-typedef struct ml_cpu_block {
-	struct ml_cpu_block *c_this;
-
-	struct gdt c_gdt;
-	struct gdt_ptr c_gdt_ptr;
-
-	struct idt_ptr c_idt_ptr;
-	unsigned int c_cpu_id;
-
-	struct cpu_data *c_data;
-} ml_cpu_block;
-
-#define ml_cpu_pause() __asm__ __volatile__("hlt")
-#define ml_cpu_relax() __asm__ __volatile__("pause")
-
-#define ml_int_disable() __asm__ __volatile__("cli")
-#define ml_int_enable() __asm__ __volatile__("sti")
-
-extern int ml_init_bootcpu(void);
-
-extern int ml_cpu_block_init(ml_cpu_block *p);
-extern int ml_cpu_block_use(ml_cpu_block *p);
-
-/* defined in cpu_ctrl.S */
-extern void ml_halt_cpu(void);
-extern ml_cpu_block *ml_this_cpu(void);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

arch/x86_64/include/mango/machine/init.h

@@ -1,26 +0,0 @@
-#ifndef MANGO_X86_64_INIT_H_
-#define MANGO_X86_64_INIT_H_
-
-#include <stdint.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define __X2(x) #x
-#define __X(x) __X2(x)
-
-#define __define_initcall(fn, id) \
-	static initcall_t __initcall_##fn##id __used \
-	__section(".initcall" __X(id) ".init") = (fn)
-
-extern int ml_init(uintptr_t arg);
-
-extern const struct framebuffer_varinfo *bootfb_varinfo(void);
-extern const struct framebuffer_fixedinfo *bootfb_fixedinfo(void);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

arch/x86_64/include/mango/machine/irq.h

@@ -1,5 +0,0 @@
-#ifndef MANGO_X86_64_IRQ_H_
-#define MANGO_X86_64_IRQ_H_
-
-
-#endif

arch/x86_64/include/mango/machine/panic.h

@@ -1,12 +0,0 @@
-#ifndef MANGO_X86_64_PANIC_H_
-#define MANGO_X86_64_PANIC_H_
-
-#include <stdint.h>
-
-struct cpu_context;
-
-extern void ml_print_cpu_state(struct cpu_context *ctx);
-extern void ml_print_stack_trace(uintptr_t ip);
-extern void ml_print_stack_trace_irq(struct cpu_context *ctx);
-
-#endif

arch/x86_64/include/mango/machine/pmap.h

@@ -1,9 +0,0 @@
-#ifndef MANGO_X86_64_PMAP_H_
-#define MANGO_X86_64_PMAP_H_
-
-#include <arch/paging.h>
-
-typedef pml4t_ptr_t ml_pmap_t;
-typedef uint64_t ml_pfn_t;
-
-#endif

arch/x86_64/include/mango/machine/thread.h

@@ -1,10 +0,0 @@
-#ifndef MANGO_X86_64_THREAD_H_
-#define MANGO_X86_64_THREAD_H_
-
-#include <mango/sched.h>
-
-extern void switch_to(struct thread *from, struct thread *to);
-extern void prepare_stack(uintptr_t ip, uintptr_t *sp);
-extern void user_jump(uintptr_t ip, uintptr_t sp);
-
-#endif

arch/x86_64/init.c

@@ -1,23 +1,39 @@
-#include "arch/serial.h"
-
 #include <arch/e820.h>
 #include <arch/pit.h>
-#include <mango/arg.h>
+#include <arch/serial.h>
-#include <mango/clock.h>
+#include <arch/vgacon.h>
-#include <mango/console.h>
+#include <kernel/arg.h>
-#include <mango/cpu.h>
+#include <kernel/bsp.h>
-#include <mango/init.h>
+#include <kernel/clock.h>
-#include <mango/libc/stdio.h>
+#include <kernel/console.h>
-#include <mango/machine/cpu.h>
+#include <kernel/cpu.h>
-#include <mango/memblock.h>
+#include <kernel/init.h>
-#include <mango/object.h>
+#include <kernel/libc/stdio.h>
-#include <mango/percpu.h>
+#include <kernel/machine/cpu.h>
-#include <mango/pmap.h>
+#include <kernel/machine/random.h>
-#include <mango/printk.h>
+#include <kernel/memblock.h>
-#include <mango/types.h>
+#include <kernel/object.h>
-#include <mango/vm.h>
+#include <kernel/percpu.h>
+#include <kernel/pmap.h>
+#include <kernel/printk.h>
+#include <kernel/types.h>
+#include <kernel/util.h>
+#include <kernel/vm.h>
 
-#define PTR32(x) ((void *)((uintptr_t)(x)))
+#define PTR32(x)            ((void *)((uintptr_t)(x)))
+
+/* the physical address of the start of the memblock heap.
+ * this is an arbirary value; the heap can start anywhere in memory.
+ * any reserved areas of memory (the kernel, bsp, bios data, etc) are
+ * automatically taken into account.
+ * HOWEVER, this value will dictate how much physical memory is required for
+ * the kernel to boot successfully.
+ * the value of 16MiB (0x1000000) means that all heap allocations will be
+ * above 16MiB, leaving the area below free for DMA operations.
+ * this value CAN be reduced all the way to zero to minimise the amount of
+ * memory required to boot, but this may leave you with no DMA memory available.
+ */
+#define MEMBLOCK_HEAP_START 0x1000000
 
 static ml_cpu_block g_bootstrap_cpu = {0};
 
@@ -30,9 +46,9 @@ static void bootstrap_cpu_init(void)
 	ml_cpu_block_use(&g_bootstrap_cpu);
 }
 
-static void early_vm_init(void)
+static void early_vm_init(uintptr_t reserve_end)
 {
-	uintptr_t alloc_start = VM_KERNEL_VOFFSET;
+	uintptr_t alloc_start = VM_KERNEL_VOFFSET + MEMBLOCK_HEAP_START;
 	/* boot code mapped 2 GiB of memory from
 	   VM_KERNEL_VOFFSET */
 	uintptr_t alloc_end = VM_KERNEL_VOFFSET + 0x7fffffff;
@@ -42,21 +58,22 @@ static void early_vm_init(void)
 	       alloc_start,
 	       alloc_end);
 
-	memblock_reserve(0x00, (uintptr_t)__pend);
+	memblock_reserve(0x00, reserve_end);
 	printk("memblock: reserved bios+kernel at [0x%016llx-0x%016llx]",
 	       0,
-	       (uintptr_t)__pend);
+	       reserve_end);
 }
 
 void early_console_init(void)
 {
 	const char *dest = arg_value("kernel.early-console");
 	if (!dest) {
-		dest = "ttyS0";
+		dest = "tty0";
 	}
 
 	if (!strcmp(dest, "tty0")) {
 		/* show log messages on VGA */
+		vgacon_init();
 	} else if (!strcmp(dest, "ttyS0")) {
 		/* write log messages to serial port */
 		early_serialcon_init(115200);
@@ -69,6 +86,23 @@ static void use_uniprocessor_topology(void)
 	cpu_set_online(0);
 }
 
+static void find_bsp(multiboot_info_t *mb, struct boot_module *out)
+{
+	memset(out, 0x0, sizeof *out);
+
+	printk("modules=%u: %llx", mb->mods_count, mb->mods_addr);
+
+	multiboot_module_t *mods = PTR32(mb->mods_addr);
+	size_t nr_mods = mb->mods_count;
+
+	if (nr_mods < 1) {
+		return;
+	}
+
+	out->mod_base = mods[0].mod_start;
+	out->mod_size = mods[0].mod_end - mods[0].mod_start;
+}
+
 int ml_init(uintptr_t arg)
 {
 	multiboot_info_t *mb = (multiboot_info_t *)arg;
@@ -82,7 +116,28 @@ int ml_init(uintptr_t arg)
 
 	print_kernel_banner();
 
-	early_vm_init();
+	struct boot_module bsp;
+	find_bsp(mb, &bsp);
+	bsp_set_location(&bsp);
+
+	uintptr_t reserve_end = (uintptr_t)__pend;
+	if (bsp.mod_base + bsp.mod_size > reserve_end) {
+		reserve_end = bsp.mod_base + bsp.mod_size;
+	}
+
+	if (ml_hwrng_available()) {
+		printk("cpu: ardware RNG available");
+		uint64_t seed = ml_hwrng_generate();
+		printk("cpu: RNG seed=%zx", seed);
+		init_random(seed);
+	} else {
+		printk("cpu: hardware RNG unavailable");
+		uint64_t seed = 0xeddc4c8a679dc23f;
+		printk("cpu: RNG seed=%zx", seed);
+		init_random(seed);
+	}
+
+	early_vm_init(reserve_end);
 
 	e820_scan(PTR32(mb->mmap_addr), mb->mmap_length);
 
@@ -101,16 +156,20 @@ int ml_init(uintptr_t arg)
 	put_cpu(this_cpu);
 
 	struct vm_zone_descriptor vm_zones[] = {
-		{.zd_id = VM_ZONE_DMA,
+		{
-	         .zd_node = 0,
+			.zd_id = VM_ZONE_DMA,
-	         .zd_name = "dma",
+			.zd_node = 0,
-	         .zd_base = 0x00,
+			.zd_name = "dma",
-	         .zd_limit = 0xffffff},
+			.zd_base = 0x00,
-		{.zd_id = VM_ZONE_NORMAL,
+			.zd_limit = 0xffffff,
-	         .zd_node = 0,
+		},
-	         .zd_name = "normal",
+		{
-	         .zd_base = 0x1000000,
+			.zd_id = VM_ZONE_NORMAL,
-	         .zd_limit = UINTPTR_MAX},
+			.zd_node = 0,
+			.zd_name = "normal",
+			.zd_base = 0x1000000,
+			.zd_limit = UINTPTR_MAX,
+		},
 	};
 
 	vm_bootstrap(vm_zones, sizeof vm_zones / sizeof vm_zones[0]);

arch/x86_64/initcall.c

@@ -1,4 +1,4 @@
-#include <mango/init.h>
+#include <kernel/init.h>
 
 extern char __initcall0_start[];
 extern char __initcall1_start[];

arch/x86_64/irq.c

@@ -1,14 +1,22 @@
 #include <arch/irq.h>
+#include <arch/msr.h>
 #include <arch/ports.h>
-#include <mango/cpu.h>
+#include <kernel/cpu.h>
-#include <mango/libc/string.h>
+#include <kernel/libc/string.h>
-#include <mango/machine/cpu.h>
+#include <kernel/machine/cpu.h>
-#include <mango/machine/irq.h>
+#include <kernel/machine/irq.h>
-#include <mango/panic.h>
+#include <kernel/panic.h>
-#include <mango/sched.h>
+#include <kernel/sched.h>
+#include <kernel/syscall.h>
 #include <stddef.h>
 
-#define MAX_ISR_HANDLERS 16
+#define MAX_ISR_HANDLERS  16
+
+#define PF_PRESENT        0x01u
+#define PF_WRITE          0x02u
+#define PF_USER           0x04u
+#define PF_RESERVED_WRITE 0x08u
+#define PF_IFETCH         0x10u
 
 extern void syscall_gate(void);
 extern uintptr_t pf_faultptr(void);
@@ -20,6 +28,27 @@ static struct idt idt;
 static int idt_initialised = 0;
 static uintptr_t int_entry_points[NR_IDT_ENTRIES];
 
+static void set_syscall_gate(uintptr_t rip)
+{
+	uint64_t user_cs = 0x13;
+	uint64_t kernel_cs = 0x08;
+
+	uintptr_t star_reg = 0xC0000081;
+	uintptr_t lstar_reg = 0xC0000082;
+	uintptr_t sfmask_reg = 0xC0000084;
+
+	uint64_t selectors = 0;
+	selectors |= (user_cs) << 48;
+	selectors |= (kernel_cs) << 32;
+
+	/* disable interrupts */
+	uint64_t flag_mask = 0x200;
+
+	wrmsr(star_reg, selectors);
+	wrmsr(lstar_reg, rip);
+	wrmsr(sfmask_reg, flag_mask);
+}
+
 static void set_idt_gate(
 	struct idt *idt,
 	uint8_t index,
@@ -39,7 +68,7 @@ static void set_idt_gate(
 	idt->i_entries[index].reserved = 0;
 }
 
-static void gpf_handler(struct cpu_context *regs)
+static void gpf_handler(struct ml_cpu_context *regs)
 {
 	int ext = regs->err_no & 1;
 	int table = (regs->err_no >> 1) & 0x03;
@@ -55,43 +84,35 @@ static void gpf_handler(struct cpu_context *regs)
 		regs->rip);
 }
 
-static void pf_handler(struct cpu_context *regs)
+static void pf_handler(struct ml_cpu_context *regs)
 {
+	enum pmap_fault_flags fault_flags = 0;
+
+	(regs->err_no & PF_PRESENT) && (fault_flags |= PMAP_FAULT_PRESENT);
+	(regs->err_no & PF_USER) && (fault_flags |= PMAP_FAULT_USER);
+	(regs->err_no & PF_WRITE) && (fault_flags |= PMAP_FAULT_WRITE);
+	(regs->err_no & PF_IFETCH) && (fault_flags |= PMAP_FAULT_IFETCH);
+	(regs->err_no & PF_RESERVED_WRITE)
+		&& (fault_flags |= PMAP_FAULT_BADCFG);
+
+	virt_addr_t fault_ptr = pf_faultptr();
+
+	ml_int_enable();
+	kern_status_t status = pmap_handle_fault(fault_ptr, fault_flags);
+	ml_int_disable();
+
+	if (status == KERN_OK) {
+		return;
+	}
+
 	panic_irq(
 		regs,
 		"page fault (%016llx %016llx %016llx)",
-		pf_faultptr(),
+		fault_ptr,
 		regs->rip,
 		regs->err_no);
 }
 
-#if 0
-static void set_syscall_gate(uintptr_t rip)
-{
-	/* sysret adds 0x10 to this to get cs, and 0x8 to get ss
-	 * note that the CPU should force the RPL to 3 when loading
-	 * the selector by using user_cs | 3. However, this doesn't happen
-	 * in certain scenarios (specifically, QEMU + KVM on a Ryzen 5 1600X). */
-	uint64_t user_cs = 0x13;
-	uint64_t kernel_cs = 0x8;
-
-	uintptr_t star_reg = 0xC0000081;
-	uintptr_t lstar_reg = 0xC0000082;
-	uintptr_t sfmask_reg = 0xC0000084;
-
-	uint64_t selectors = 0;
-	selectors |= (user_cs) << 48;
-	selectors |= (kernel_cs) << 32;
-
-	/* disable interrupts */
-	uint64_t flag_mask = 0x200;
-
-	write_msr(star_reg, selectors);
-	write_msr(lstar_reg, rip);
-	write_msr(sfmask_reg, flag_mask);
-}
-#endif
-
 static void init_pic(void)
 {
 	// Remap the PIC
@@ -129,6 +150,8 @@ int idt_init(struct idt_ptr *ptr)
 		init_global_idt();
 	}
 
+	set_syscall_gate((uintptr_t)syscall_gate);
+
 	ptr->i_limit = sizeof(idt) - 1;
 	ptr->i_base = (uintptr_t)&idt;
 
@@ -141,7 +164,7 @@ int idt_load(struct idt_ptr *ptr)
 	return 0;
 }
 
-void isr_dispatch(struct cpu_context *regs)
+void isr_dispatch(struct ml_cpu_context *regs)
 {
 	int_hook h = isr_handlers[regs->int_no];
 	if (h) {
@@ -160,7 +183,7 @@ void irq_ack(unsigned int vec)
 	outportb(0x20, 0x20);
 }
 
-void irq_dispatch(struct cpu_context *regs)
+void irq_dispatch(struct ml_cpu_context *regs)
 {
 	end_charge_period();
 
@@ -178,8 +201,40 @@ void irq_dispatch(struct cpu_context *regs)
 	start_charge_period();
 }
 
-void syscall_dispatch(struct cpu_context *regs)
+void syscall_dispatch(struct ml_cpu_context *regs)
 {
+	unsigned int sysid = regs->rax;
+	virt_addr_t syscall_impl = syscall_get_function(sysid);
+
+	if (syscall_impl == 0) {
+		regs->rax = KERN_UNSUPPORTED;
+		return;
+	}
+
+#define SYSCALL_SIGNATURE(...)                                                 \
+	intptr_t (*__VA_ARGS__)(                                               \
+		uintptr_t,                                                     \
+		uintptr_t,                                                     \
+		uintptr_t,                                                     \
+		uintptr_t,                                                     \
+		uintptr_t,                                                     \
+		uintptr_t,                                                     \
+		uintptr_t,                                                     \
+		uintptr_t)
+
+	SYSCALL_SIGNATURE(fn) = (SYSCALL_SIGNATURE())syscall_impl;
+
+	ml_int_enable();
+	regs->rax
+		= fn(regs->rdi,
+	             regs->rsi,
+	             regs->rdx,
+	             regs->r12,
+	             regs->r8,
+	             regs->r9,
+	             regs->r13,
+	             regs->r14);
+	ml_int_disable();
 }
 
 void hook_irq(enum irq_vector vec, struct irq_hook *hook)
@@ -194,263 +249,263 @@ void unhook_irq(enum irq_vector vec, struct irq_hook *hook)
 	queue_delete(hook_queue, &hook->irq_entry);
 }
 
-extern void _isr0();
+extern void _isr0(void);
-extern void _isr1();
+extern void _isr1(void);
-extern void _isr2();
+extern void _isr2(void);
-extern void _isr3();
+extern void _isr3(void);
-extern void _isr4();
+extern void _isr4(void);
-extern void _isr5();
+extern void _isr5(void);
-extern void _isr6();
+extern void _isr6(void);
-extern void _isr7();
+extern void _isr7(void);
-extern void _isr8();
+extern void _isr8(void);
-extern void _isr9();
+extern void _isr9(void);
-extern void _isr10();
+extern void _isr10(void);
-extern void _isr11();
+extern void _isr11(void);
-extern void _isr12();
+extern void _isr12(void);
-extern void _isr13();
+extern void _isr13(void);
-extern void _isr14();
+extern void _isr14(void);
-extern void _isr15();
+extern void _isr15(void);
-extern void _isr16();
+extern void _isr16(void);
-extern void _isr17();
+extern void _isr17(void);
-extern void _isr18();
+extern void _isr18(void);
-extern void _isr19();
+extern void _isr19(void);
-extern void _isr20();
+extern void _isr20(void);
-extern void _isr21();
+extern void _isr21(void);
-extern void _isr22();
+extern void _isr22(void);
-extern void _isr23();
+extern void _isr23(void);
-extern void _isr24();
+extern void _isr24(void);
-extern void _isr25();
+extern void _isr25(void);
-extern void _isr26();
+extern void _isr26(void);
-extern void _isr27();
+extern void _isr27(void);
-extern void _isr28();
+extern void _isr28(void);
-extern void _isr29();
+extern void _isr29(void);
-extern void _isr30();
+extern void _isr30(void);
-extern void _isr31();
+extern void _isr31(void);
 
-extern void _irq0();
+extern void _irq0(void);
-extern void _irq1();
+extern void _irq1(void);
-extern void _irq2();
+extern void _irq2(void);
-extern void _irq3();
+extern void _irq3(void);
-extern void _irq4();
+extern void _irq4(void);
-extern void _irq5();
+extern void _irq5(void);
-extern void _irq6();
+extern void _irq6(void);
-extern void _irq7();
+extern void _irq7(void);
-extern void _irq8();
+extern void _irq8(void);
-extern void _irq9();
+extern void _irq9(void);
-extern void _irq10();
+extern void _irq10(void);
-extern void _irq11();
+extern void _irq11(void);
-extern void _irq12();
+extern void _irq12(void);
-extern void _irq13();
+extern void _irq13(void);
-extern void _irq14();
+extern void _irq14(void);
-extern void _irq15();
+extern void _irq15(void);
-extern void _irq16();
+extern void _irq16(void);
-extern void _irq17();
+extern void _irq17(void);
-extern void _irq18();
+extern void _irq18(void);
-extern void _irq19();
+extern void _irq19(void);
-extern void _irq20();
+extern void _irq20(void);
-extern void _irq21();
+extern void _irq21(void);
-extern void _irq22();
+extern void _irq22(void);
-extern void _irq23();
+extern void _irq23(void);
-extern void _irq24();
+extern void _irq24(void);
-extern void _irq25();
+extern void _irq25(void);
-extern void _irq26();
+extern void _irq26(void);
-extern void _irq27();
+extern void _irq27(void);
-extern void _irq28();
+extern void _irq28(void);
-extern void _irq29();
+extern void _irq29(void);
-extern void _irq30();
+extern void _irq30(void);
-extern void _irq31();
+extern void _irq31(void);
-extern void _irq32();
+extern void _irq32(void);
-extern void _irq33();
+extern void _irq33(void);
-extern void _irq34();
+extern void _irq34(void);
-extern void _irq35();
+extern void _irq35(void);
-extern void _irq36();
+extern void _irq36(void);
-extern void _irq37();
+extern void _irq37(void);
-extern void _irq38();
+extern void _irq38(void);
-extern void _irq39();
+extern void _irq39(void);
-extern void _irq40();
+extern void _irq40(void);
-extern void _irq41();
+extern void _irq41(void);
-extern void _irq42();
+extern void _irq42(void);
-extern void _irq43();
+extern void _irq43(void);
-extern void _irq44();
+extern void _irq44(void);
-extern void _irq45();
+extern void _irq45(void);
-extern void _irq46();
+extern void _irq46(void);
-extern void _irq47();
+extern void _irq47(void);
-extern void _irq48();
+extern void _irq48(void);
-extern void _irq49();
+extern void _irq49(void);
-extern void _irq50();
+extern void _irq50(void);
-extern void _irq51();
+extern void _irq51(void);
-extern void _irq52();
+extern void _irq52(void);
-extern void _irq53();
+extern void _irq53(void);
-extern void _irq54();
+extern void _irq54(void);
-extern void _irq55();
+extern void _irq55(void);
-extern void _irq56();
+extern void _irq56(void);
-extern void _irq57();
+extern void _irq57(void);
-extern void _irq58();
+extern void _irq58(void);
-extern void _irq59();
+extern void _irq59(void);
-extern void _irq60();
+extern void _irq60(void);
-extern void _irq61();
+extern void _irq61(void);
-extern void _irq62();
+extern void _irq62(void);
-extern void _irq63();
+extern void _irq63(void);
-extern void _irq64();
+extern void _irq64(void);
-extern void _irq65();
+extern void _irq65(void);
-extern void _irq66();
+extern void _irq66(void);
-extern void _irq67();
+extern void _irq67(void);
-extern void _irq68();
+extern void _irq68(void);
-extern void _irq69();
+extern void _irq69(void);
-extern void _irq70();
+extern void _irq70(void);
-extern void _irq71();
+extern void _irq71(void);
-extern void _irq72();
+extern void _irq72(void);
-extern void _irq73();
+extern void _irq73(void);
-extern void _irq74();
+extern void _irq74(void);
-extern void _irq75();
+extern void _irq75(void);
-extern void _irq76();
+extern void _irq76(void);
-extern void _irq77();
+extern void _irq77(void);
-extern void _irq78();
+extern void _irq78(void);
-extern void _irq79();
+extern void _irq79(void);
-extern void _irq80();
+extern void _irq80(void);
-extern void _irq81();
+extern void _irq81(void);
-extern void _irq82();
+extern void _irq82(void);
-extern void _irq83();
+extern void _irq83(void);
-extern void _irq84();
+extern void _irq84(void);
-extern void _irq85();
+extern void _irq85(void);
-extern void _irq86();
+extern void _irq86(void);
-extern void _irq87();
+extern void _irq87(void);
-extern void _irq88();
+extern void _irq88(void);
-extern void _irq89();
+extern void _irq89(void);
-extern void _irq90();
+extern void _irq90(void);
-extern void _irq91();
+extern void _irq91(void);
-extern void _irq92();
+extern void _irq92(void);
-extern void _irq93();
+extern void _irq93(void);
-extern void _irq94();
+extern void _irq94(void);
-extern void _irq95();
+extern void _irq95(void);
-extern void _irq96();
+extern void _irq96(void);
-extern void _irq97();
+extern void _irq97(void);
-extern void _irq98();
+extern void _irq98(void);
-extern void _irq99();
+extern void _irq99(void);
-extern void _irq100();
+extern void _irq100(void);
-extern void _irq101();
+extern void _irq101(void);
-extern void _irq102();
+extern void _irq102(void);
-extern void _irq103();
+extern void _irq103(void);
-extern void _irq104();
+extern void _irq104(void);
-extern void _irq105();
+extern void _irq105(void);
-extern void _irq106();
+extern void _irq106(void);
-extern void _irq107();
+extern void _irq107(void);
-extern void _irq108();
+extern void _irq108(void);
-extern void _irq109();
+extern void _irq109(void);
-extern void _irq110();
+extern void _irq110(void);
-extern void _irq111();
+extern void _irq111(void);
-extern void _irq112();
+extern void _irq112(void);
-extern void _irq113();
+extern void _irq113(void);
-extern void _irq114();
+extern void _irq114(void);
-extern void _irq115();
+extern void _irq115(void);
-extern void _irq116();
+extern void _irq116(void);
-extern void _irq117();
+extern void _irq117(void);
-extern void _irq118();
+extern void _irq118(void);
-extern void _irq119();
+extern void _irq119(void);
-extern void _irq120();
+extern void _irq120(void);
-extern void _irq121();
+extern void _irq121(void);
-extern void _irq122();
+extern void _irq122(void);
-extern void _irq123();
+extern void _irq123(void);
-extern void _irq124();
+extern void _irq124(void);
-extern void _irq125();
+extern void _irq125(void);
-extern void _irq126();
+extern void _irq126(void);
-extern void _irq127();
+extern void _irq127(void);
-extern void _irq128();
+extern void _irq128(void);
-extern void _irq129();
+extern void _irq129(void);
-extern void _irq130();
+extern void _irq130(void);
-extern void _irq131();
+extern void _irq131(void);
-extern void _irq132();
+extern void _irq132(void);
-extern void _irq133();
+extern void _irq133(void);
-extern void _irq134();
+extern void _irq134(void);
-extern void _irq135();
+extern void _irq135(void);
-extern void _irq136();
+extern void _irq136(void);
-extern void _irq137();
+extern void _irq137(void);
-extern void _irq138();
+extern void _irq138(void);
-extern void _irq139();
+extern void _irq139(void);
-extern void _irq140();
+extern void _irq140(void);
-extern void _irq141();
+extern void _irq141(void);
-extern void _irq142();
+extern void _irq142(void);
-extern void _irq143();
+extern void _irq143(void);
-extern void _irq144();
+extern void _irq144(void);
-extern void _irq145();
+extern void _irq145(void);
-extern void _irq146();
+extern void _irq146(void);
-extern void _irq147();
+extern void _irq147(void);
-extern void _irq148();
+extern void _irq148(void);
-extern void _irq149();
+extern void _irq149(void);
-extern void _irq150();
+extern void _irq150(void);
-extern void _irq151();
+extern void _irq151(void);
-extern void _irq152();
+extern void _irq152(void);
-extern void _irq153();
+extern void _irq153(void);
-extern void _irq154();
+extern void _irq154(void);
-extern void _irq155();
+extern void _irq155(void);
-extern void _irq156();
+extern void _irq156(void);
-extern void _irq157();
+extern void _irq157(void);
-extern void _irq158();
+extern void _irq158(void);
-extern void _irq159();
+extern void _irq159(void);
-extern void _irq160();
+extern void _irq160(void);
-extern void _irq161();
+extern void _irq161(void);
-extern void _irq162();
+extern void _irq162(void);
-extern void _irq163();
+extern void _irq163(void);
-extern void _irq164();
+extern void _irq164(void);
-extern void _irq165();
+extern void _irq165(void);
-extern void _irq166();
+extern void _irq166(void);
-extern void _irq167();
+extern void _irq167(void);
-extern void _irq168();
+extern void _irq168(void);
-extern void _irq169();
+extern void _irq169(void);
-extern void _irq170();
+extern void _irq170(void);
-extern void _irq171();
+extern void _irq171(void);
-extern void _irq172();
+extern void _irq172(void);
-extern void _irq173();
+extern void _irq173(void);
-extern void _irq174();
+extern void _irq174(void);
-extern void _irq175();
+extern void _irq175(void);
-extern void _irq176();
+extern void _irq176(void);
-extern void _irq177();
+extern void _irq177(void);
-extern void _irq178();
+extern void _irq178(void);
-extern void _irq179();
+extern void _irq179(void);
-extern void _irq180();
+extern void _irq180(void);
-extern void _irq181();
+extern void _irq181(void);
-extern void _irq182();
+extern void _irq182(void);
-extern void _irq183();
+extern void _irq183(void);
-extern void _irq184();
+extern void _irq184(void);
-extern void _irq185();
+extern void _irq185(void);
-extern void _irq186();
+extern void _irq186(void);
-extern void _irq187();
+extern void _irq187(void);
-extern void _irq188();
+extern void _irq188(void);
-extern void _irq189();
+extern void _irq189(void);
-extern void _irq190();
+extern void _irq190(void);
-extern void _irq191();
+extern void _irq191(void);
-extern void _irq192();
+extern void _irq192(void);
-extern void _irq193();
+extern void _irq193(void);
-extern void _irq194();
+extern void _irq194(void);
-extern void _irq195();
+extern void _irq195(void);
-extern void _irq196();
+extern void _irq196(void);
-extern void _irq197();
+extern void _irq197(void);
-extern void _irq198();
+extern void _irq198(void);
-extern void _irq199();
+extern void _irq199(void);
-extern void _irq200();
+extern void _irq200(void);
-extern void _irq201();
+extern void _irq201(void);
-extern void _irq202();
+extern void _irq202(void);
-extern void _irq203();
+extern void _irq203(void);
-extern void _irq204();
+extern void _irq204(void);
-extern void _irq205();
+extern void _irq205(void);
-extern void _irq206();
+extern void _irq206(void);
-extern void _irq207();
+extern void _irq207(void);
-extern void _irq208();
+extern void _irq208(void);
-extern void _irq209();
+extern void _irq209(void);
-extern void _irq210();
+extern void _irq210(void);
-extern void _irq211();
+extern void _irq211(void);
-extern void _irq212();
+extern void _irq212(void);
-extern void _irq213();
+extern void _irq213(void);
-extern void _irq214();
+extern void _irq214(void);
-extern void _irq215();
+extern void _irq215(void);
-extern void _irq216();
+extern void _irq216(void);
-extern void _irq217();
+extern void _irq217(void);
-extern void _irq218();
+extern void _irq218(void);
-extern void _irq219();
+extern void _irq219(void);
-extern void _irq220();
+extern void _irq220(void);
-extern void _irq221();
+extern void _irq221(void);
-extern void _irq222();
+extern void _irq222(void);
-extern void _irq223();
+extern void _irq223(void);
 
 static uintptr_t int_entry_points[NR_IDT_ENTRIES] = {
 	[0] = (uintptr_t)_isr0,     [1] = (uintptr_t)_isr1,

arch/x86_64/irqvec.S

@@ -332,80 +332,115 @@ IRQ 223, 255
 
 isr_common_stub:
 	PUSH_REGS
-	
+
+	# When ISR occurs in Ring 3, CPU sets %ss (and other non-code selectors)
+	# to 0.
+	mov %ss, %ax
+	cmp $0, %ax
+	jne isr_skipgs1
+
+	mov $0x10, %ax
+	mov %ax, %ss
+	swapgs
+
+isr_skipgs1:
 	mov %rsp, %rdi
 	call isr_dispatch
-	
+
 	POP_REGS
 	add $16, %rsp
+
+	cmpq $0x1b, 32(%rsp)
+	jne isr_skipgs2
+
+	swapgs
+
+isr_skipgs2:
+
 	iretq
-	
+
 
 	.global irq_common_stub
 	.type irq_common_stub, @function
 
 irq_common_stub:
 	PUSH_REGS
-	
+
+	# When IRQ occurs in Ring 3, CPU sets %ss (and other non-code selectors)
+	# to 0.
+	mov %ss, %ax
+	cmp $0, %ax
+	jne irq_skipgs1
+
+	mov $0x10, %ax
+	mov %ax, %ss
+	swapgs
+
+irq_skipgs1:
 	mov %rsp, %rdi
 	call irq_dispatch
-	
+
+
 	POP_REGS
 	add $16, %rsp
+
+	cmpq $0x1b, 32(%rsp)
+	jne isr_skipgs2
+
+	swapgs
+
+irq_skipgs2:
+
 	iretq
-	
+
 
 	.global syscall_gate
 	.type syscall_gate, @function
-	
+
 	.extern syscall_dispatch
 	.type syscall_dispatch, @function
-	
+
 syscall_gate:
 	swapgs
-	movq %rsp, %gs:20       # GS+20 = rsp2 in the current TSS block (user stack storage)
+	movq %rsp, %gs:94       # GS+20 = rsp2 in the current TSS block (user stack storage)
-	movq %gs:4, %rsp	# GS+4  = rsp0 in the current TSS block (per-thread kstack)
+	movq %gs:78, %rsp	# GS+4  = rsp0 in the current TSS block (per-thread kstack)
-	
+
-	# start building a pf_cpu_context
+	# start building a ml_cpu_context
 	pushq $0x1b
-	pushq %gs:20
+	pushq %gs:94
 	push %r11
 	push $0x23
 	push %rcx
-	
+
 	pushq $0
 	pushq $0x80
-	
+
 	PUSH_REGS
-	
+
 	mov %rsp, %rdi
-	
+
-	# switch back to user gs while in syscall_dispatch. Interrupts are enabled in syscall_dispatch,
-	# and if the task gets pre-empted, the incoming task will expect %gs to have its usermode value.
-	swapgs
-	
 	call syscall_dispatch
-	
+
 	POP_REGS
-	
+
 	add $16, %rsp
 	pop %rcx
 	add $8, %rsp
 	pop %r11
 	add $16, %rsp
-	
+
+	movq %gs:94, %rsp       # GS+20 = rsp2 in the current TSS block
+
 	swapgs
-	movq %gs:20, %rsp       # GS+20 = rsp2 in the current TSS block
+
-	swapgs
-	
 	# back to usermode
 	sysretq
-	
+
 
 	.global pf_faultptr
 	.type pf_faultptr, @function
 pf_faultptr:
 	mov %cr2, %rax
 	ret
-	
+
-	
+

arch/x86_64/panic.c

@@ -1,24 +1,28 @@
-#include "mango/machine/panic.h"
-#include "mango/vm.h"
-#include <mango/printk.h>
-#include <mango/libc/stdio.h>
 #include <arch/irq.h>
+#include <kernel/address-space.h>
+#include <kernel/libc/stdio.h>
+#include <kernel/machine/cpu.h>
+#include <kernel/machine/panic.h>
+#include <kernel/printk.h>
+#include <kernel/sched.h>
+#include <kernel/task.h>
+#include <kernel/vm.h>
 
-#define R_CF 0
+#define R_CF  0
-#define R_PF 2
+#define R_PF  2
-#define R_AF 4
+#define R_AF  4
-#define R_ZF 6
+#define R_ZF  6
-#define R_SF 7
+#define R_SF  7
-#define R_TF 8
+#define R_TF  8
-#define R_IF 9
+#define R_IF  9
-#define R_DF 10
+#define R_DF  10
-#define R_OF 11
+#define R_OF  11
-#define R_NT 14
+#define R_NT  14
-#define R_VM 17
+#define R_VM  17
-#define R_AC 18
+#define R_AC  18
 #define R_VIF 19
 #define R_VIP 20
-#define R_ID 21
+#define R_ID  21
 #define R_MAX 21
 
 struct stack_frame {
@@ -82,7 +86,11 @@ static void print_rflags(uintptr_t rflags)
 		if (rflags & (1 << i)) {
 			const char *name = pf_rfl_name(i);
 			if (name) {
-				buf_i += snprintf(buf + buf_i, sizeof(buf) - buf_i, " %s", name);
+				buf_i += snprintf(
+					buf + buf_i,
+					sizeof(buf) - buf_i,
+					" %s",
+					name);
 			}
 		}
 	}
@@ -91,30 +99,43 @@ static void print_rflags(uintptr_t rflags)
 	printk(buf);
 }
 
-void ml_print_cpu_state(struct cpu_context *ctx)
+void ml_print_cpu_state(struct ml_cpu_context *ctx)
 {
 	printk("cpu state:");
 	if (ctx) {
 		printk("  rax %016llx rbx %016llx rcx %016llx",
-		       ctx->rax, ctx->rbx, ctx->rcx);
+		       ctx->rax,
+		       ctx->rbx,
+		       ctx->rcx);
 		printk("  rdx %016llx rsi %016llx rdi %016llx",
-		       ctx->rdx, ctx->rsi, ctx->rdi);
+		       ctx->rdx,
+		       ctx->rsi,
+		       ctx->rdi);
 		printk("  rsp %016llx rbp %016llx r8  %016llx",
-		       ctx->rsp, ctx->rbp, ctx->r8);
+		       ctx->rsp,
+		       ctx->rbp,
+		       ctx->r8);
 		printk("  r9  %016llx r10 %016llx r11 %016llx",
-		       ctx->r9, ctx->r10, ctx->r11);
+		       ctx->r9,
+		       ctx->r10,
+		       ctx->r11);
 		printk("  r12 %016llx r13 %016llx r14 %016llx",
-		       ctx->r12, ctx->r13, ctx->r14);
+		       ctx->r12,
+		       ctx->r13,
+		       ctx->r14);
 		printk("  r15 %016llx rip %016llx cs  %04x ss %04x",
-		       ctx->r15, ctx->rip, ctx->cs, ctx->ss);
+		       ctx->r15,
+		       ctx->rip,
+		       ctx->cs,
+		       ctx->ss);
 		print_rflags(ctx->rflags);
 	}
 
 	uintptr_t cr0 = 0, cr2 = 0, cr3 = 0, cr4 = 0;
-	asm volatile("mov %%cr0, %%rax" : "=a" (cr0));
+	asm volatile("mov %%cr0, %%rax" : "=a"(cr0));
-	asm volatile("mov %%cr2, %%rax" : "=a" (cr2));
+	asm volatile("mov %%cr2, %%rax" : "=a"(cr2));
-	asm volatile("mov %%cr3, %%rax" : "=a" (cr3));
+	asm volatile("mov %%cr3, %%rax" : "=a"(cr3));
-	asm volatile("mov %%cr4, %%rax" : "=a" (cr4));
+	asm volatile("mov %%cr4, %%rax" : "=a"(cr4));
 	printk("  cr0 %016llx cr2 %016llx", cr0, cr2);
 	printk("  cr3 %016llx cr4 %016llx", cr3, cr4);
 }
@@ -124,56 +145,88 @@ static void print_stack_item(uintptr_t addr)
 	if (!addr) {
 		return;
 	}
-	
+
 	char buf[64];
 	size_t i = 0;
-	
+
 	i += snprintf(buf, sizeof(buf), "  [<%p>] ", addr);
-	
+
 	size_t offset = 0;
 	char name[128];
 	int found = -1;
-	
+
 	if (found == 0 && name[0] != '\0') {
-		i += snprintf(buf + i, sizeof(buf) - i, "%s+0x%lx", name, offset);
+		i += snprintf(
+			buf + i,
+			sizeof(buf) - i,
+			"%s+0x%lx",
+			name,
+			offset);
 	} else {
 		i += snprintf(buf + i, sizeof(buf) - i, "?");
 	}
-	
+
 	printk("%s", buf);
 }
 
-static void print_stack_trace(uintptr_t ip, uintptr_t *bp)
+static bool read_stack_frame(
+	struct address_space *space,
+	uintptr_t bp,
+	struct stack_frame *out)
 {
-	struct stack_frame *stk = (struct stack_frame *)bp;
+	if (bp >= VM_PAGEMAP_BASE) {
+		*out = *(struct stack_frame *)out;
+		return true;
+	}
+
+	if (!space) {
+		return false;
+	}
+
+	size_t tmp;
+	kern_status_t status
+		= address_space_read(space, bp, sizeof *out, out, &tmp);
+	return status == KERN_OK;
+}
+
+static void print_stack_trace(
+	struct address_space *space,
+	uintptr_t ip,
+	uintptr_t bp)
+{
+	struct stack_frame stk;
+	if (!read_stack_frame(space, bp, &stk)) {
+		return;
+	}
 	printk("call trace:");
-	
+
 	print_stack_item(ip);
 
 	int max_frames = 10, current_frame = 0;
-	while (1) {
+	while (current_frame < max_frames) {
-		if (!vm_virt_to_phys(stk) ||
+		uintptr_t addr = stk.rip;
-		    bp == NULL ||
+		print_stack_item(addr);
-		    current_frame > max_frames) {
+
+		bp = stk.rbp;
+		if (!read_stack_frame(space, bp, &stk)) {
 			break;
 		}
-		
-		uintptr_t addr = stk->rip;
-		print_stack_item(addr);
-		
-		stk = (struct stack_frame *)stk->rbp;
 		current_frame++;
 	}
 }
 
 void ml_print_stack_trace(uintptr_t ip)
 {
-	uintptr_t *bp;
+	struct task *task = current_task();
-	asm volatile("mov %%rbp, %0" : "=r" (bp));
+	struct address_space *space = task ? task->t_address_space : NULL;
-	print_stack_trace(ip, bp);
+	uintptr_t bp;
+	asm volatile("mov %%rbp, %0" : "=r"(bp));
+	print_stack_trace(space, ip, bp);
 }
 
-void ml_print_stack_trace_irq(struct cpu_context *ctx)
+void ml_print_stack_trace_irq(struct ml_cpu_context *ctx)
 {
-	print_stack_trace(ctx->rip, (uintptr_t *)ctx->rbp);
+	struct task *task = current_task();
+	struct address_space *space = task ? task->t_address_space : NULL;
+	print_stack_trace(space, ctx->rip, ctx->rbp);
 }

arch/x86_64/pit.c

@@ -1,8 +1,8 @@
 #include <arch/irq.h>
 #include <arch/ports.h>
-#include <mango/clock.h>
+#include <kernel/clock.h>
-#include <mango/cpu.h>
+#include <kernel/cpu.h>
-#include <mango/printk.h>
+#include <kernel/printk.h>
 
 #define PIT_COUNTER0  0x40
 #define PIT_CMD       0x43

arch/x86_64/pmap.c

@@ -1,20 +1,25 @@
-#include <mango/types.h>
+#include <kernel/address-space.h>
-#include <mango/memblock.h>
+#include <kernel/compiler.h>
-#include <mango/vm.h>
+#include <kernel/libc/stdio.h>
-#include <mango/printk.h>
+#include <kernel/memblock.h>
+#include <kernel/pmap.h>
+#include <kernel/printk.h>
+#include <kernel/sched.h>
+#include <kernel/task.h>
+#include <kernel/types.h>
+#include <kernel/vm-object.h>
+#include <kernel/vm.h>
 #include <mango/status.h>
-#include <mango/compiler.h>
-#include <mango/pmap.h>
 
 /* some helpful datasize constants */
-#define C_1GiB 0x40000000ULL
+#define C_1GiB          0x40000000ULL
-#define C_2GiB (2 * C_1GiB)
+#define C_2GiB          (2 * C_1GiB)
 
-#define BAD_INDEX ((unsigned int)-1)
+#define BAD_INDEX       ((unsigned int)-1)
-#define PTR_TO_ENTRY(x) (((x) & ~VM_PAGE_MASK) | PTE_PRESENT | PTE_RW)
+#define PTR_TO_ENTRY(x) (((x) & ~VM_PAGE_MASK) | PTE_PRESENT | PTE_RW | PTE_USR)
 #define ENTRY_TO_PTR(x) ((x) & ~VM_PAGE_MASK)
 
-#define PFN(x) ((x) >> VM_PAGE_SHIFT)
+#define PFN(x)          ((x) >> VM_PAGE_SHIFT)
 
 static int can_use_gbpages = 0;
 static pmap_t kernel_pmap;
@@ -33,24 +38,26 @@ static size_t ps_size(enum page_size ps)
 	}
 }
 
-static pmap_t alloc_pmap()
+static pmap_t alloc_pmap(void)
 {
 	struct pml4t *p = kzalloc(sizeof *p, 0);
 	return vm_virt_to_phys(p);
 }
 
-static pte_t make_pte(pfn_t pfn, enum vm_prot prot, enum page_size size)
+static pte_t make_pte(pfn_t pfn, vm_prot_t prot, enum page_size size)
 {
 	pte_t v = pfn;
 
 	switch (size) {
 	case PS_1G:
-		/* pfn_t is in terms of 4KiB pages, convert to 1GiB page frame number */
+		/* pfn_t is in terms of 4KiB pages, convert to 1GiB page frame
+		 * number */
 		pfn >>= 18;
 		v = (pfn & 0x3FFFFF) << 30;
 		break;
 	case PS_2M:
-		/* pfn_t is in terms of 4KiB pages, convert to 2MiB page frame number */
+		/* pfn_t is in terms of 4KiB pages, convert to 2MiB page frame
+		 * number */
 		pfn >>= 9;
 		v = (pfn & 0x7FFFFFFF) << 21;
 		break;
@@ -129,14 +136,15 @@ static void delete_pdir(phys_addr_t pd)
 	kfree(pdir);
 }
 
-static kern_status_t do_pmap_add(pmap_t pmap, void *p, pfn_t pfn, enum vm_prot prot, enum page_size size)
+static kern_status_t do_pmap_add(
+	pmap_t pmap,
+	virt_addr_t pv,
+	pfn_t pfn,
+	vm_prot_t prot,
+	enum page_size size)
 {
-	uintptr_t pv = (uintptr_t)p;
+	unsigned int pml4t_index = BAD_INDEX, pdpt_index = BAD_INDEX,
-	unsigned int
+		     pd_index = BAD_INDEX, pt_index = BAD_INDEX;
-		pml4t_index = BAD_INDEX,
-		pdpt_index = BAD_INDEX,
-		pd_index = BAD_INDEX,
-		pt_index = BAD_INDEX;
 
 	switch (size) {
 	case PS_4K:
@@ -171,16 +179,19 @@ static kern_status_t do_pmap_add(pmap_t pmap, void *p, pfn_t pfn, enum vm_prot p
 	struct pdpt *pdpt = NULL;
 	if (!pml4t->p_entries[pml4t_index]) {
 		pdpt = kzalloc(sizeof *pdpt, 0);
-		pml4t->p_entries[pml4t_index] = PTR_TO_ENTRY(vm_virt_to_phys(pdpt));
+		pml4t->p_entries[pml4t_index]
+			= PTR_TO_ENTRY(vm_virt_to_phys(pdpt));
 	} else {
-		pdpt = vm_phys_to_virt(ENTRY_TO_PTR(pml4t->p_entries[pml4t_index]));
+		pdpt = vm_phys_to_virt(
+			ENTRY_TO_PTR(pml4t->p_entries[pml4t_index]));
 	}
 
 	/* if we're mapping a 1GiB page, we stop here */
 	if (size == PS_1G) {
 		if (pdpt->p_entries[pdpt_index] != 0) {
 			/* this slot points to a pdir, delete it.
-			   if this slot points to a hugepage, this does nothing */
+			   if this slot points to a hugepage, this does nothing
+			 */
 			delete_pdir(pdpt->p_entries[pdpt_index]);
 		}
 
@@ -189,22 +200,25 @@ static kern_status_t do_pmap_add(pmap_t pmap, void *p, pfn_t pfn, enum vm_prot p
 		return KERN_OK;
 	}
 
-
 	/* 3. traverse PDPT, get PDIR (optional, 4K and 2M only) */
 	struct pdir *pdir = NULL;
-	if (!pdpt->p_entries[pdpt_index] || pdpt->p_pages[pdpt_index] & PTE_PAGESIZE) {
+	if (!pdpt->p_entries[pdpt_index]
+	    || pdpt->p_pages[pdpt_index] & PTE_PAGESIZE) {
 		/* entry is null, or points to a hugepage */
 		pdir = kzalloc(sizeof *pdir, 0);
-		pdpt->p_entries[pdpt_index] = PTR_TO_ENTRY(vm_virt_to_phys(pdir));
+		pdpt->p_entries[pdpt_index]
+			= PTR_TO_ENTRY(vm_virt_to_phys(pdir));
 	} else {
-		pdir = vm_phys_to_virt(ENTRY_TO_PTR(pdpt->p_entries[pdpt_index]));
+		pdir = vm_phys_to_virt(
+			ENTRY_TO_PTR(pdpt->p_entries[pdpt_index]));
 	}
 
 	/* if we're mapping a 2MiB page, we stop here */
 	if (size == PS_2M) {
 		if (pdir->p_entries[pd_index] != 0) {
 			/* this slot points to a ptab, delete it.
-			   if this slot points to a hugepage, this does nothing */
+			   if this slot points to a hugepage, this does nothing
+			 */
 			delete_ptab(pdir->p_entries[pd_index]);
 		}
 
@@ -214,7 +228,8 @@ static kern_status_t do_pmap_add(pmap_t pmap, void *p, pfn_t pfn, enum vm_prot p
 
 	/* 4. traverse PDIR, get PTAB (optional, 4K only) */
 	struct ptab *ptab = NULL;
-	if (!pdir->p_entries[pd_index] || pdir->p_pages[pd_index] & PTE_PAGESIZE) {
+	if (!pdir->p_entries[pd_index]
+	    || pdir->p_pages[pd_index] & PTE_PAGESIZE) {
 		/* entry is null, or points to a hugepage */
 		ptab = kzalloc(sizeof *ptab, 0);
 		pdir->p_entries[pd_index] = PTR_TO_ENTRY(vm_virt_to_phys(ptab));
@@ -234,7 +249,8 @@ pmap_t get_kernel_pmap(void)
 void pmap_bootstrap(void)
 {
 	can_use_gbpages = gigabyte_pages();
-	printk("pmap: gigabyte pages %sabled", can_use_gbpages == 1 ? "en" : "dis");
+	printk("pmap: gigabyte pages %sabled",
+	       can_use_gbpages == 1 ? "en" : "dis");
 	enable_nx();
 	printk("pmap: NX protection enabled");
 
@@ -249,18 +265,21 @@ void pmap_bootstrap(void)
 	/* map 2GiB at the end of the address space to
 	   replace the mapping created by start_32 and allow access to
 	   the kernel and memblock-allocated data. */
-	uintptr_t vbase = VM_KERNEL_VOFFSET;
+	virt_addr_t vbase = VM_KERNEL_VOFFSET;
 	for (size_t i = 0; i < C_2GiB; i += hugepage_sz) {
-		do_pmap_add(kernel_pmap,
+		do_pmap_add(
-			    (void *)(vbase + i),
+			kernel_pmap,
-			    PFN(i),
+			vbase + i,
-			    VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXEC | VM_PROT_SVR,
+			PFN(i),
-			    hugepage);
+			VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXEC
+				| VM_PROT_SVR,
+			hugepage);
 	}
 
 	phys_addr_t pmem_limit = 0x0;
 	struct memblock_iter it;
-	for_each_mem_range(&it, 0x00, UINTPTR_MAX) {
+	for_each_mem_range(&it, 0x00, UINTPTR_MAX)
+	{
 		if (it.it_limit > pmem_limit) {
 			pmem_limit = it.it_limit;
 		}
@@ -268,10 +287,13 @@ void pmap_bootstrap(void)
 
 	vbase = VM_PAGEMAP_BASE;
 	for (size_t i = 0; i < pmem_limit; i += hugepage_sz) {
-		do_pmap_add(kernel_pmap,
+		do_pmap_add(
-			    (void *)(vbase + i),
+			kernel_pmap,
-			    PFN(i),
+			vbase + i,
-			    VM_PROT_READ | VM_PROT_WRITE | VM_PROT_SVR | VM_PROT_NOCACHE, hugepage);
+			PFN(i),
+			VM_PROT_READ | VM_PROT_WRITE | VM_PROT_SVR
+				| VM_PROT_NOCACHE,
+			hugepage);
 	}
 
 	pmap_switch(kernel_pmap);
@@ -279,15 +301,87 @@ void pmap_bootstrap(void)
 
 pmap_t pmap_create(void)
 {
-	return 0;
+	pmap_t pmap = alloc_pmap();
+	pmap_t kernel_pmap = get_kernel_pmap();
+
+	struct pml4t *pml4t = vm_phys_to_virt(pmap);
+	struct pml4t *kernel_pml4t = vm_phys_to_virt(kernel_pmap);
+
+	for (unsigned int i = 256; i < 512; i++) {
+		pml4t->p_entries[i] = kernel_pml4t->p_entries[i];
+	}
+
+	return pmap;
 }
 
 void pmap_destroy(pmap_t pmap)
 {
-
 }
 
-kern_status_t pmap_add(pmap_t pmap, void *p, pfn_t pfn, enum vm_prot prot, enum pmap_flags flags)
+static void log_fault(virt_addr_t fault_addr, enum pmap_fault_flags flags)
+{
+	char flag_str[128] = {0};
+	size_t p = 0;
+
+	if (flags & PMAP_FAULT_PRESENT) {
+		p += snprintf(flag_str + p, sizeof flag_str - p, " PRESENT");
+	} else {
+		p += snprintf(flag_str + p, sizeof flag_str - p, " MISSING");
+	}
+
+	if (flags & PMAP_FAULT_USER) {
+		p += snprintf(flag_str + p, sizeof flag_str - p, " USER");
+	} else {
+		p += snprintf(flag_str + p, sizeof flag_str - p, " SVR");
+	}
+
+	if (flags & PMAP_FAULT_WRITE) {
+		p += snprintf(flag_str + p, sizeof flag_str - p, " WRITE");
+	} else {
+		p += snprintf(flag_str + p, sizeof flag_str - p, " READ");
+	}
+
+	if (flags & PMAP_FAULT_IFETCH) {
+		p += snprintf(flag_str + p, sizeof flag_str - p, " IFETCH");
+	}
+
+	if (flags & PMAP_FAULT_BADCFG) {
+		p += snprintf(flag_str + p, sizeof flag_str - p, " BADCFG");
+	}
+
+	printk("pmap: fault at 0x%llx (%s)", fault_addr, flag_str);
+}
+
+kern_status_t pmap_handle_fault(
+	virt_addr_t fault_addr,
+	enum pmap_fault_flags flags)
+{
+	// log_fault(fault_addr, flags);
+
+	if (flags & PMAP_FAULT_PRESENT) {
+		return KERN_FATAL_ERROR;
+	}
+
+	struct task *task = current_task();
+	if (!task) {
+		return KERN_FATAL_ERROR;
+	}
+
+	struct address_space *space = task->t_address_space;
+	if (!space) {
+		return KERN_FATAL_ERROR;
+	}
+
+	/* this must be called with `space` unlocked. */
+	return address_space_demand_map(space, fault_addr, flags);
+}
+
+kern_status_t pmap_add(
+	pmap_t pmap,
+	virt_addr_t p,
+	pfn_t pfn,
+	vm_prot_t prot,
+	enum pmap_flags flags)
 {
 	enum page_size ps = PS_4K;
 	if (flags & PMAP_HUGEPAGE) {
@@ -297,17 +391,23 @@ kern_status_t pmap_add(pmap_t pmap, void *p, pfn_t pfn, enum vm_prot prot, enum
 	return do_pmap_add(pmap, p, pfn, prot, ps);
 }
 
-kern_status_t pmap_add_block(pmap_t pmap, void *p, pfn_t pfn, size_t len, enum vm_prot prot, enum pmap_flags flags)
+kern_status_t pmap_add_block(
+	pmap_t pmap,
+	virt_addr_t p,
+	pfn_t pfn,
+	size_t len,
+	vm_prot_t prot,
+	enum pmap_flags flags)
 {
 	return KERN_OK;
 }
 
-kern_status_t pmap_remove(pmap_t pmap, void *p)
+kern_status_t pmap_remove(pmap_t pmap, virt_addr_t p)
 {
 	return KERN_OK;
 }
 
-kern_status_t pmap_remove_range(pmap_t pmap, void *p, size_t len)
+kern_status_t pmap_remove_range(pmap_t pmap, virt_addr_t p, size_t len)
 {
 	return KERN_OK;
 }

arch/x86_64/ports.c

@@ -1,39 +1,50 @@
 #include <stdint.h>
 
-uint8_t inportb(uint16_t port) {
+uint8_t inportb(uint16_t port)
-    uint8_t data;
+{
-    __asm__ __volatile__("inb %1, %0" : "=a"(data) : "dN"(port));
+	uint8_t data;
-    return data;
+	__asm__ __volatile__("inb %1, %0" : "=a"(data) : "dN"(port));
+	return data;
 }
 
-void outportb(uint16_t port, uint8_t data) {
+void outportb(uint16_t port, uint8_t data)
-    __asm__ __volatile__("outb %1, %0" : : "dN"(port), "a"(data));
+{
+	__asm__ __volatile__("outb %1, %0" : : "dN"(port), "a"(data));
 }
 
-uint16_t inportw(uint16_t port) {
+uint16_t inportw(uint16_t port)
-    uint16_t data;
+{
-    __asm__ __volatile__("inw %1, %0" : "=a"(data) : "dN"(port));
+	uint16_t data;
-    return data;
+	__asm__ __volatile__("inw %1, %0" : "=a"(data) : "dN"(port));
+	return data;
 }
 
-void outportw(uint16_t port, uint16_t data) {
+void outportw(uint16_t port, uint16_t data)
-    __asm__ __volatile__("outw %1, %0" : : "dN"(port), "a"(data));
+{
+	__asm__ __volatile__("outw %1, %0" : : "dN"(port), "a"(data));
 }
 
-uint32_t inportl(uint16_t port) {
+uint32_t inportl(uint16_t port)
-    uint32_t data;
+{
-    __asm__ __volatile__("inl %%dx, %%eax" : "=a"(data) : "dN"(port));
+	uint32_t data;
-    return data;
+	__asm__ __volatile__("inl %%dx, %%eax" : "=a"(data) : "dN"(port));
+	return data;
 }
 
-void outportl(uint16_t port, uint32_t data) {
+void outportl(uint16_t port, uint32_t data)
-    __asm__ __volatile__("outl %%eax, %%dx" : : "dN"(port), "a"(data));
+{
+	__asm__ __volatile__("outl %%eax, %%dx" : : "dN"(port), "a"(data));
 }
 
-void outportsw(uint16_t port, void *data, uint32_t size) {
+void outportsw(uint16_t port, void *data, uint32_t size)
-    __asm__ __volatile__("rep outsw" : "+S"(data), "+c"(size) : "d"(port));
+{
+	__asm__ __volatile__("rep outsw" : "+S"(data), "+c"(size) : "d"(port));
 }
 
-void inportsw(uint16_t port, unsigned char *data, unsigned long size) {
+void inportsw(uint16_t port, unsigned char *data, unsigned long size)
-    __asm__ __volatile__("rep insw" : "+D"(data), "+c"(size) : "d"(port) : "memory");
+{
+	__asm__ __volatile__("rep insw"
+	                     : "+D"(data), "+c"(size)
+	                     : "d"(port)
+	                     : "memory");
 }

arch/x86_64/random.S

@@ -0,0 +1,43 @@
+.code64
+
+.global ml_hwrng_available
+.type ml_hwrng_available, @function
+
+ml_hwrng_available:
+	push %rbp
+	mov %rsp, %rbp
+	push %rbx
+	push %rdx
+
+	mov $1, %eax
+	mov $0, %ecx
+
+	cpuid
+
+	shr $30, %ecx
+	and $1, %ecx
+	mov %ecx, %eax
+
+	pop %rdx
+	pop %rbx
+	pop %rbp
+	ret
+
+.global ml_hwrng_generate
+.type ml_hwrng_generate, @function
+
+ml_hwrng_generate:
+	push %rbp
+	mov %rsp, %rbp
+
+	mov $100, %rcx
+.retry:
+	rdrand %rax
+	jc .done
+	loop .retry
+
+.fail:
+	mov $0, %rax
+.done:
+	pop %rbp
+	ret

arch/x86_64/serial.c

@@ -1,8 +1,8 @@
 #include <arch/irq.h>
 #include <arch/ports.h>
 #include <arch/serial.h>
-#include <mango/libc/stdio.h>
+#include <kernel/libc/stdio.h>
-#include <mango/printk.h>
+#include <kernel/printk.h>
 
 #define COM1 0x3F8
 #define COM2 0x2F8

arch/x86_64/targets.cmake

@@ -1,7 +1,21 @@
 find_program(QEMU qemu-system-x86_64)
+set(DEBUG_SESSION ${CMAKE_SOURCE_DIR}/tools/kernel-debug/debug_session.sh)
+set(DEBUG_CFG_GDB ${CMAKE_SOURCE_DIR}/tools/kernel-debug/gdb_session_init)
+set(DEBUG_CFG_LLDB ${CMAKE_SOURCE_DIR}/tools/kernel-debug/lldb_session_init)
 
 add_custom_target(run
 	USES_TERMINAL
 	COMMAND ${QEMU}
 		-kernel $<TARGET_FILE:${kernel_exe_name}>
 		-serial stdio)
+
+# LLDB DEPENDS ON AN ELF64 EXECUTABLE.
+# .DBG FILE MUST NOT BE PATCHED WITH E64PATCH!
+add_custom_target(debug
+	USES_TERMINAL
+	COMMAND ${DEBUG_SESSION}
+		${DEBUG_CFG_GDB}
+		${DEBUG_CFG_LLDB}
+		${QEMU} -kernel $<TARGET_FILE:${kernel_exe_name}>
+			-S -s
+			-monitor stdio)

arch/x86_64/thread.c

@@ -1,12 +1,24 @@
-#include <mango/machine/thread.h>
+#include <kernel/machine/cpu.h>
+#include <kernel/machine/thread.h>
 
+#define MAX_REG_ARGS 6
+#define REG_ARG_0    rdi
+#define REG_ARG_1    rsi
+#define REG_ARG_2    rdx
+#define REG_ARG_3    rcx
+#define REG_ARG_4    r8
+#define REG_ARG_5    r9
+
+/* this is the context information restored by ml_thread_switch.
+ * since ml_thread_switch only jumps to kernel-mode, IRETQ isn't used,
+ * and the extra register values needed by IRETQ aren't present. */
 struct thread_ctx {
 	uint64_t r15, r14, r13, r12, r11, r10, r9, r8;
 	uint64_t rdi, rsi, rbp, unused_rsp, rbx, rdx, rcx, rax;
 	uint64_t rfl;
 } __packed;
 
-void prepare_stack(uintptr_t ip, uintptr_t *sp)
+void ml_thread_prepare_kernel_context(uintptr_t ip, uintptr_t *sp)
 {
 	(*sp) -= sizeof(uintptr_t);
 	uintptr_t *dest_ip = (uintptr_t *)(*sp);
@@ -18,3 +30,50 @@ void prepare_stack(uintptr_t ip, uintptr_t *sp)
 
 	ctx->rfl = 0x202;
 }
+
+extern kern_status_t ml_thread_prepare_user_context(
+	virt_addr_t ip,
+	virt_addr_t user_sp,
+	virt_addr_t *kernel_sp,
+	const uintptr_t *args,
+	size_t nr_args)
+{
+	(*kernel_sp) -= sizeof(struct ml_cpu_context);
+
+	struct ml_cpu_context *ctx = (struct ml_cpu_context *)(*kernel_sp);
+	memset(ctx, 0x0, sizeof *ctx);
+	ctx->rip = ip;
+	ctx->rsp = user_sp;
+	ctx->ss = 0x1b;
+	ctx->cs = 0x23;
+	ctx->rflags = 0x202;
+	ctx->rdi = 0;   // arg 0
+	ctx->rsi = 0;   // arg 1
+
+	for (size_t i = 0; i < nr_args; i++) {
+		switch (i) {
+		case 0:
+			ctx->REG_ARG_0 = args[i];
+			break;
+		case 1:
+			ctx->REG_ARG_1 = args[i];
+			break;
+		case 2:
+			ctx->REG_ARG_2 = args[i];
+			break;
+		case 3:
+			ctx->REG_ARG_3 = args[i];
+			break;
+		case 4:
+			ctx->REG_ARG_4 = args[i];
+			break;
+		case 5:
+			ctx->REG_ARG_5 = args[i];
+			break;
+		default:
+			return KERN_INVALID_ARGUMENT;
+		}
+	}
+
+	return KERN_OK;
+}

arch/x86_64/thread_switch.S

@@ -1,13 +1,13 @@
-	.code64
+.code64
 
-	.extern THREAD_sp
+.extern THREAD_sp
-	
+
-	.global switch_to
+.global ml_thread_switch
-	.type switch_to, @function
+.type ml_thread_switch, @function
 
 	// %rdi = (struct thread *) current thread.
 	// %rsi = (struct thread *) next thread.
-switch_to:
+ml_thread_switch:
 	pushfq
 
 	push %rax
@@ -26,7 +26,7 @@ switch_to:
 	push %r13
 	push %r14
 	push %r15
-	
+
 	movq %rsp, THREAD_sp(%rdi)
 	movq THREAD_sp(%rsi), %rsp
 
@@ -50,3 +50,28 @@ switch_to:
 	popfq
 
 	ret
+
+
+.global ml_thread_switch_user
+.type ml_thread_switch_user, @function
+ml_thread_switch_user:
+	pop %r15
+	pop %r14
+	pop %r13
+	pop %r12
+	pop %r11
+	pop %r10
+	pop %r9
+	pop %r8
+	pop %rdi
+	pop %rsi
+	pop %rbp
+	add $8, %rsp
+	pop %rbx
+	pop %rdx
+	pop %rcx
+	pop %rax
+
+	add $16, %rsp
+	swapgs
+	iretq

arch/x86_64/toolchain.cmake

@@ -6,6 +6,8 @@ set(CMAKE_C_COMPILER   x86_64-elf-gcc)
 set(CMAKE_ASM_COMPILER   x86_64-elf-gcc)
 set(CMAKE_CXX_COMPILER x86_64-elf-g++)
 
+set(CMAKE_STRIP x86_64-elf-strip)
+
 set(CMAKE_C_COMPILER_WORKS TRUE)
 set(CMAKE_CXX_COMPILER_WORKS TRUE)
 

arch/x86_64/tss.c

@@ -0,0 +1,43 @@
+#include <arch/gdt.h>
+#include <arch/tss.h>
+#include <kernel/libc/string.h>
+
+static void tss_flush(int index)
+{
+	index *= sizeof(struct gdt_entry);
+	index |= 3;
+	asm volatile("mov %0, %%eax; ltr %%ax" ::"r"(index));
+}
+
+void tss_init(struct tss *tss, struct tss_ptr *ptr)
+{
+	memset(tss, 0x0, sizeof *tss);
+
+	ptr->tss_base = (uint64_t)tss;
+	ptr->tss_limit = (uint16_t)sizeof *tss;
+}
+
+void tss_load(struct tss *tss)
+{
+	tss_flush(TSS_GDT_INDEX);
+}
+
+virt_addr_t tss_get_kstack(struct tss *tss)
+{
+	return tss->rsp0;
+}
+
+virt_addr_t tss_get_ustack(struct tss *tss)
+{
+	return tss->rsp2;
+}
+
+void tss_set_kstack(struct tss *tss, virt_addr_t sp)
+{
+	tss->rsp0 = sp;
+}
+
+void tss_set_ustack(struct tss *tss, virt_addr_t sp)
+{
+	tss->rsp2 = sp;
+}

arch/x86_64/vga.c

@@ -0,0 +1,126 @@
+#include <arch/irq.h>
+#include <arch/ports.h>
+#include <arch/serial.h>
+#include <kernel/libc/stdio.h>
+#include <kernel/machine/vm.h>
+#include <kernel/printk.h>
+
+struct vga_console {
+	uint16_t *vga_framebuffer;
+	unsigned int vga_cursor_x, vga_cursor_y;
+	unsigned int vga_screen_width, vga_screen_height;
+	uint8_t vga_attrib;
+};
+
+static struct vga_console vga_con = {
+	.vga_attrib = 0x0F,
+	.vga_screen_width = 80,
+	.vga_screen_height = 25,
+	.vga_framebuffer = (uint16_t *)0xffffffff800b8000,
+};
+
+static void vga_console_clear(struct vga_console *con)
+{
+	size_t len = con->vga_screen_width * con->vga_screen_height;
+
+	for (size_t i = 0; i < len; i++) {
+		con->vga_framebuffer[i] = (uint16_t)con->vga_attrib << 8;
+	}
+
+	con->vga_cursor_x = 0;
+	con->vga_cursor_y = 0;
+}
+
+static void vga_console_show_cursor(struct vga_console *con)
+{
+	size_t start = 0, end = 15;
+
+	outportb(0x3D4, 0x0A);
+	outportb(0x3D5, (inportb(0x3D5) & 0xC0) | start);
+
+	outportb(0x3D4, 0x0B);
+	outportb(0x3D5, (inportb(0x3D5) & 0xE0) | end);
+}
+
+static void vga_console_update_cursor(struct vga_console *con)
+{
+	uint16_t pos
+		= con->vga_cursor_y * con->vga_screen_width + con->vga_cursor_x;
+
+	outportb(0x3D4, 0x0F);
+	outportb(0x3D5, (uint8_t)(pos & 0xFF));
+	outportb(0x3D4, 0x0E);
+	outportb(0x3D5, (uint8_t)((pos >> 8) & 0xFF));
+}
+
+static void vga_console_scroll(struct vga_console *con)
+{
+	uint16_t *src = &con->vga_framebuffer[con->vga_screen_width];
+	uint16_t *dest = &con->vga_framebuffer[0];
+	size_t len = (con->vga_screen_height - 1) * con->vga_screen_width
+	           * sizeof *con->vga_framebuffer;
+
+	memcpy(dest, src, len);
+
+	dest = &con->vga_framebuffer
+	                [(con->vga_screen_height - 1) * con->vga_screen_width];
+	len = con->vga_screen_width;
+	for (size_t i = 0; i < len; i++) {
+		dest[i] = (uint16_t)con->vga_attrib << 8;
+	}
+
+	con->vga_cursor_x = 0;
+	con->vga_cursor_y = con->vga_screen_height - 1;
+}
+
+static void vga_console_putc(struct vga_console *con, char c)
+{
+	switch (c) {
+	case '\n':
+		con->vga_cursor_x = 0;
+		con->vga_cursor_y++;
+		break;
+	case '\r':
+		con->vga_cursor_x = 0;
+		break;
+	default:
+		con->vga_framebuffer
+			[con->vga_cursor_y * con->vga_screen_width
+		         + con->vga_cursor_x]
+			= ((uint16_t)con->vga_attrib << 8) | c;
+		con->vga_cursor_x++;
+		break;
+	}
+
+	if (con->vga_cursor_x >= con->vga_screen_width) {
+		con->vga_cursor_x = 0;
+		con->vga_cursor_y++;
+	}
+
+	if (con->vga_cursor_y >= con->vga_screen_height) {
+		vga_console_scroll(con);
+	}
+}
+
+static void vgacon_write(struct console *con, const char *s, unsigned int len)
+{
+	for (unsigned int i = 0; i < len; i++) {
+		vga_console_putc(&vga_con, s[i]);
+	}
+
+	vga_console_update_cursor(&vga_con);
+}
+
+static struct console vgacon = {
+	.c_name = "vgacon",
+	.c_flags = CON_BOOT,
+	.c_write = vgacon_write,
+	.c_lock = SPIN_LOCK_INIT,
+};
+
+void vgacon_init(void)
+{
+	vga_console_clear(&vga_con);
+	vga_console_show_cursor(&vga_con);
+	console_register(&vgacon);
+}

ds/bitmap.c

@@ -1,5 +1,5 @@
-#include <mango/libc/string.h>
+#include <kernel/bitmap.h>
-#include <mango/bitmap.h>
+#include <kernel/libc/string.h>
 
 void bitmap_zero(unsigned long *map, unsigned long nbits)
 {
@@ -25,7 +25,7 @@ void bitmap_set(unsigned long *map, unsigned long bit)
 void bitmap_clear(unsigned long *map, unsigned long bit)
 {
 	unsigned long index = bit / BITS_PER_WORD;
-	unsigned long offset = bit & (BITS_PER_WORD - 1);
+	unsigned long offset = (BITS_PER_WORD - bit - 1) & (BITS_PER_WORD - 1);
 	unsigned long mask = 1ul << offset;
 
 	map[index] &= ~mask;
@@ -38,7 +38,6 @@ bool bitmap_check(unsigned long *map, unsigned long bit)
 	unsigned long mask = 1ul << offset;
 
 	return (map[index] & mask) != 0 ? true : false;
-
 }
 
 unsigned int bitmap_count_set(unsigned long *map, unsigned long nbits)

ds/btree.c

@@ -57,7 +57,7 @@
    provide a comparator function.
 */ 
    
-#include <mango/btree.h>
+#include <kernel/btree.h>
 #include <stddef.h>
 
 #define MAX(a, b) ((a) > (b) ? (a) : (b))

ds/queue.c

@@ -1,4 +1,4 @@
-#include <mango/queue.h>
+#include <kernel/queue.h>
 
 size_t queue_length(struct queue *q)
 {

ds/ringbuffer.c

@@ -1,5 +1,6 @@
-#include <mango/ringbuffer.h>
+#include <kernel/ringbuffer.h>
-#include <mango/sched.h>
+#include <kernel/sched.h>
+#include <kernel/vm.h>
 
 size_t ringbuffer_unread(struct ringbuffer *ring_buffer)
 {
@@ -9,7 +10,7 @@ size_t ringbuffer_unread(struct ringbuffer *ring_buffer)
 
 	if (ring_buffer->r_read_ptr > ring_buffer->r_write_ptr) {
 		return (ring_buffer->r_size - ring_buffer->r_read_ptr)
-			+ ring_buffer->r_write_ptr;
+		     + ring_buffer->r_write_ptr;
 	} else {
 		return (ring_buffer->r_write_ptr - ring_buffer->r_read_ptr);
 	}
@@ -25,7 +26,7 @@ size_t ringbuffer_avail(struct ringbuffer *ring_buffer)
 		return ring_buffer->r_read_ptr - ring_buffer->r_write_ptr - 1;
 	} else {
 		return (ring_buffer->r_size - ring_buffer->r_write_ptr)
-			+ ring_buffer->r_read_ptr - 1;
+		     + ring_buffer->r_read_ptr - 1;
 	}
 }
 
@@ -45,7 +46,11 @@ static inline void increment_write(struct ringbuffer *ring_buffer)
 	}
 }
 
-size_t ringbuffer_read(struct ringbuffer *ring_buffer, size_t size, void *p, mango_flags_t flags)
+size_t ringbuffer_read(
+	struct ringbuffer *ring_buffer,
+	size_t size,
+	void *p,
+	mango_flags_t flags)
 {
 	if (!ring_buffer) {
 		return 0;
@@ -58,7 +63,9 @@ size_t ringbuffer_read(struct ringbuffer *ring_buffer, size_t size, void *p, man
 	while (collected < size) {
 		spin_lock_irqsave(&ring_buffer->r_lock, &lock_flags);
 		while (ringbuffer_unread(ring_buffer) > 0 && collected < size) {
-			buffer[collected] = ring_buffer->r_buffer[ring_buffer->r_read_ptr];
+			buffer[collected]
+				= ring_buffer
+			                  ->r_buffer[ring_buffer->r_read_ptr];
 			increment_read(ring_buffer);
 			collected++;
 		}
@@ -66,7 +73,9 @@ size_t ringbuffer_read(struct ringbuffer *ring_buffer, size_t size, void *p, man
 		wakeup_queue(&ring_buffer->r_wait_writers);
 
 		if (flags & S_NOBLOCK) {
-			spin_unlock_irqrestore(&ring_buffer->r_lock, lock_flags);
+			spin_unlock_irqrestore(
+				&ring_buffer->r_lock,
+				lock_flags);
 			break;
 		}
 
@@ -86,7 +95,11 @@ size_t ringbuffer_read(struct ringbuffer *ring_buffer, size_t size, void *p, man
 	return collected;
 }
 
-size_t ringbuffer_write(struct ringbuffer *ring_buffer, size_t size, const void *p, mango_flags_t flags)
+size_t ringbuffer_write(
+	struct ringbuffer *ring_buffer,
+	size_t size,
+	const void *p,
+	mango_flags_t flags)
 {
 	if (!ring_buffer || !size) {
 		return 0;
@@ -100,7 +113,8 @@ size_t ringbuffer_write(struct ringbuffer *ring_buffer, size_t size, const void
 		spin_lock_irqsave(&ring_buffer->r_lock, &lock_flags);
 
 		while (ringbuffer_avail(ring_buffer) > 0 && written < size) {
-			ring_buffer->r_buffer[ring_buffer->r_write_ptr] = buffer[written];
+			ring_buffer->r_buffer[ring_buffer->r_write_ptr]
+				= buffer[written];
 			increment_write(ring_buffer);
 			written++;
 		}
@@ -108,7 +122,9 @@ size_t ringbuffer_write(struct ringbuffer *ring_buffer, size_t size, const void
 		wakeup_queue(&ring_buffer->r_wait_readers);
 
 		if (flags & S_NOBLOCK) {
-			spin_unlock_irqrestore(&ring_buffer->r_lock, lock_flags);
+			spin_unlock_irqrestore(
+				&ring_buffer->r_lock,
+				lock_flags);
 			break;
 		}
 

include/kernel/address-space.h

@@ -0,0 +1,163 @@
+#ifndef KERNEL_ADDRESS_SPACE_H_
+#define KERNEL_ADDRESS_SPACE_H_
+
+#include <kernel/object.h>
+#include <kernel/pmap.h>
+#include <kernel/vm.h>
+
+#define ADDRESS_SPACE_COPY_ALL ((size_t)-1)
+
+struct address_space;
+struct vm_object;
+
+struct vm_area {
+	/* the vm-object mapped into this area.
+	 * if this is NULL, the vm_area represents an area of reserved memory.
+	 * it cannot be accessed, and mapping operations with MAP_ADDRESS_ANY
+	 * will avoid the area, but fixed address mappings in this area
+	 * will succeed. */
+	struct vm_object *vma_object;
+	/* used to link to vm_object->vo_mappings */
+	struct queue_entry vma_object_entry;
+	/* the memory protection flags applied to this area */
+	vm_prot_t vma_prot;
+	/* offset in bytes to the start of the object data that was mapped */
+	off_t vma_object_offset;
+	/* used to link to address_space->s_mappings */
+	struct btree_node vma_node;
+	/* address of the first byte in this area */
+	virt_addr_t vma_base;
+	/* address of the last byte in this area */
+	virt_addr_t vma_limit;
+};
+
+struct address_space {
+	struct object s_base;
+
+	/* address of the first byte in this address space */
+	virt_addr_t s_base_address;
+	/* address of the last byte in this address space */
+	virt_addr_t s_limit_address;
+
+	/* btree of struct vm_area representing mapped vm-objects.
+	 * sibling entries cannot overlap each other. */
+	struct btree s_mappings;
+	/* btree of struct vm_area representing reserved regions of the
+	 * address space.
+	 * reserved regions will not be automatically allocated by the kernel.
+	 * sibling entries cannot overlap each other.
+	 * overlap between s_mappings and s_reserved IS allowed. */
+	struct btree s_reserved;
+
+	/* the corresponding physical address space */
+	pmap_t s_pmap;
+};
+
+extern kern_status_t address_space_type_init(void);
+extern struct address_space *address_space_cast(struct object *obj);
+
+/* create a new vm-region, optionally within a parent region.
+ * `offset` is the byte offset within the parent region where the new region
+ * should start.
+ * if no parent is specified, `offset` is the absolute virtual address of the
+ * start of the region.
+ * in both cases, `len` is the length of the new region in bytes. */
+extern kern_status_t address_space_create(
+	virt_addr_t base,
+	virt_addr_t limit,
+	struct address_space **out);
+
+/* map a vm-object into a vm-region.
+ * [region_offset,length] must fall within exactly one region, and cannot span
+ * multiple sibling regions.
+ * if [region_offset,length] falls within a child region, the map operation
+ * will be transparently redirected to the relevant region.
+ * `prot` must be allowed both by the region into which the mapping is being
+ * created AND the vm-object being mapped. */
+extern kern_status_t address_space_map(
+	struct address_space *space,
+	virt_addr_t map_address,
+	struct vm_object *object,
+	off_t object_offset,
+	size_t length,
+	vm_prot_t prot,
+	virt_addr_t *out);
+extern kern_status_t address_space_unmap(
+	struct address_space *region,
+	virt_addr_t base,
+	size_t length);
+
+/* reserve an area of the address space. the kernel will not place any
+ * new mappings in this area unless explicitly told to (i.e. by not using
+ * MAP_ADDRESS_ANY). Use MAP_ADDRESS_ANY to have the kernel allocate a region
+ * of the address space for you */
+extern kern_status_t address_space_reserve(
+	struct address_space *space,
+	virt_addr_t base,
+	size_t length,
+	virt_addr_t *out);
+/* release a previously reserved area of the address space. */
+extern kern_status_t address_space_release(
+	struct address_space *space,
+	virt_addr_t base,
+	size_t length);
+
+extern bool address_space_validate_access(
+	struct address_space *region,
+	virt_addr_t base,
+	size_t len,
+	vm_prot_t prot);
+
+/* find the mapping corresponding to the given virtual address, and page-in the
+ * necessary vm_page to allow the memory access to succeed. if the relevant
+ * vm-object page hasn't been allocated yet, it will be allocated here.
+ * this function must be called with `region` UNLOCKED and interrupts ENABLED.
+ */
+extern kern_status_t address_space_demand_map(
+	struct address_space *region,
+	virt_addr_t addr,
+	enum pmap_fault_flags flags);
+
+/* read data from the user-space area of a vm-region into a kernel-mode buffer
+ */
+extern kern_status_t address_space_read(
+	struct address_space *src_region,
+	virt_addr_t src_ptr,
+	size_t count,
+	void *dest,
+	size_t *nr_read);
+
+/* write data to the user-space area of a vm-region from a kernel-mode buffer
+ */
+extern kern_status_t address_space_write(
+	struct address_space *dst_region,
+	virt_addr_t dst_ptr,
+	size_t count,
+	const void *src,
+	size_t *nr_written);
+
+extern kern_status_t address_space_memmove(
+	struct address_space *dest_space,
+	virt_addr_t dest_ptr,
+	struct address_space *src_space,
+	virt_addr_t src_ptr,
+	size_t count,
+	size_t *nr_moved);
+
+extern kern_status_t address_space_memmove_v(
+	struct address_space *dest_space,
+	size_t dest_offset,
+	const kern_iovec_t *dest_iov,
+	size_t nr_dest_iov,
+	struct address_space *src_space,
+	size_t src_offset,
+	const kern_iovec_t *src_iov,
+	size_t nr_src_iov,
+	size_t bytes_to_move,
+	size_t *nr_bytes_moved);
+
+void address_space_dump(struct address_space *region);
+
+DEFINE_OBJECT_LOCK_FUNCTION(address_space, s_base)
+
+#endif

include/kernel/arg.h

@@ -1,8 +1,8 @@
-#ifndef MANGO_ARG_H_
+#ifndef KERNEL_ARG_H_
-#define MANGO_ARG_H_
+#define KERNEL_ARG_H_
 
+#include <mango/types.h>
 #include <stdbool.h>
-#include <mango/status.h>
 
 #define CMDLINE_MAX 4096
 

include/kernel/bitmap.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_BITMAP_H_
+#ifndef KERNEL_BITMAP_H_
-#define MANGO_BITMAP_H_
+#define KERNEL_BITMAP_H_
 
 #include <stdbool.h>
 

include/kernel/bsp.h

@@ -0,0 +1,38 @@
+#ifndef KERNEL_BSP_H_
+#define KERNEL_BSP_H_
+
+#include <kernel/compiler.h>
+#include <mango/status.h>
+#include <kernel/types.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#define BSP_MAGIC 0xcafebabe
+
+struct task;
+
+struct bsp_trailer {
+	/* these fields are stored in big endian in the package itself */
+	uint32_t bsp_magic;
+	uint64_t bsp_fs_offset;
+	uint32_t bsp_fs_len;
+	uint64_t bsp_exec_offset;
+	uint32_t bsp_exec_len;
+	uint64_t bsp_text_faddr, bsp_text_vaddr, bsp_text_size;
+	uint64_t bsp_data_faddr, bsp_data_vaddr, bsp_data_size;
+	uint64_t bsp_exec_entry;
+} __packed;
+
+struct bsp {
+	/* the values in this struct are stored in host byte order */
+	struct bsp_trailer bsp_trailer;
+	struct vm_object *bsp_vmo;
+};
+
+extern void bsp_set_location(const struct boot_module *mod);
+extern void bsp_get_location(struct boot_module *out);
+
+extern kern_status_t bsp_load(struct bsp *bsp, const struct boot_module *mod);
+extern kern_status_t bsp_launch_async(struct bsp *bsp, struct task *task);
+
+#endif

include/kernel/btree.h

@@ -20,99 +20,130 @@
     software without specific prior written permission.
  */
 
-#ifndef MANGO_BTREE_H_
+#ifndef KERNEL_BTREE_H_
-#define MANGO_BTREE_H_
+#define KERNEL_BTREE_H_
 
+#include <stdbool.h>
+#include <stddef.h>
 #include <stdint.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-/* if your custom structure contains a struct btree_node (i.e. it can be part of a btree),
+/* if your custom structure contains a struct btree_node (i.e. it can be part of
-   you can use this macro to convert a struct btree_node* to a your_type*
+   a btree), you can use this macro to convert a struct btree_node* to a
+   your_type*
 
-   @param t the name of your custom type (something that can be passed to offsetof)
+   @param t the name of your custom type (something that can be passed to
-   @param m the name of the struct btree_node member variable within your custom type.
+   offsetof)
-   @param v the struct btree_node pointer that you wish to convert. if this is NULL, NULL will be returned.
+   @param m the name of the struct btree_node member variable within your custom
+   type.
+   @param v the struct btree_node pointer that you wish to convert. if this is
+   NULL, NULL will be returned.
 */
-#define BTREE_CONTAINER(t, m, v) ((void *)((v) ? (uintptr_t)(v) - (offsetof(t, m)) : 0))
+#define BTREE_CONTAINER(t, m, v)                                               \
+	((void *)((v) ? (uintptr_t)(v) - (offsetof(t, m)) : 0))
 
 /* defines a simple node insertion function.
-   this function assumes that your nodes have simple integer keys that can be compared with the usual operators.
+   this function assumes that your nodes have simple integer keys that can be
+   compared with the usual operators.
 
    EXAMPLE:
    if you have a tree node type like this:
 
      struct my_tree_node {
              int key;
-	     struct btree_node base;
+             struct btree_node base;
      }
 
-   You would use the following call to generate an insert function for a tree with this node type:
+   You would use the following call to generate an insert function for a tree
+   with this node type:
 
-     BTREE_DEFINE_SIMPLE_INSERT(struct my_tree_node, base, key, my_tree_node_insert);
+     BTREE_DEFINE_SIMPLE_INSERT(struct my_tree_node, base, key,
+   my_tree_node_insert);
 
    Which would emit a function defined like:
 
-     static void my_tree_node_insert(struct btree *tree, struct my_tree_node *node);
+     static void my_tree_node_insert(struct btree *tree, struct my_tree_node
+   *node);
 
-   @param node_type your custom tree node type. usually a structure that contains a struct btree_node member.
+   @param node_type your custom tree node type. usually a structure that
-   @param container_node_member the name of the struct btree_node member variable within your custom type.
+   contains a struct btree_node member.
-   @param container_key_member the name of the key member variable within your custom type.
+   @param container_node_member the name of the struct btree_node member
+   variable within your custom type.
+   @param container_key_member the name of the key member variable within your
+   custom type.
    @param function_name the name of the function to generate.
 */
-#define BTREE_DEFINE_SIMPLE_INSERT(node_type, container_node_member, container_key_member, function_name) \
+#define BTREE_DEFINE_SIMPLE_INSERT(                                            \
-	void function_name(struct btree *tree, node_type *node)                                           \
+	node_type,                                                             \
-	{                                                                                                 \
+	container_node_member,                                                 \
-		if (!tree->b_root) {                                                                      \
+	container_key_member,                                                  \
-			tree->b_root = &node->container_node_member;                                      \
+	function_name)                                                         \
-			btree_insert_fixup(tree, &node->container_node_member);                           \
+	void function_name(struct btree *tree, node_type *node)                \
-			return;                                                                           \
+	{                                                                      \
-		}                                                                                         \
+		if (!tree->b_root) {                                           \
-                                                                                                          \
+			tree->b_root = &node->container_node_member;           \
-		struct btree_node *cur = tree->b_root;                                                    \
+			btree_insert_fixup(                                    \
-		while (1) {                                                                               \
+				tree,                                          \
-			node_type *cur_node = BTREE_CONTAINER(node_type, container_node_member, cur);     \
+				&node->container_node_member);                 \
-			struct btree_node *next = NULL;                                                   \
+			return;                                                \
-                                                                                                          \
+		}                                                              \
-			if (node->container_key_member > cur_node->container_key_member) {                \
+                                                                               \
-				next = btree_right(cur);                                                  \
+		struct btree_node *cur = tree->b_root;                         \
-                                                                                                          \
+		while (1) {                                                    \
-				if (!next) {                                                              \
+			node_type *cur_node = BTREE_CONTAINER(                 \
-					btree_put_right(cur, &node->container_node_member);               \
+				node_type,                                     \
-					break;                                                            \
+				container_node_member,                         \
-				}                                                                         \
+				cur);                                          \
-			} else if (node->container_key_member < cur_node->container_key_member) {         \
+			struct btree_node *next = NULL;                        \
-				next = btree_left(cur);                                                   \
+                                                                               \
-                                                                                                          \
+			if (node->container_key_member                         \
-				if (!next) {                                                              \
+			    > cur_node->container_key_member) {                \
-					btree_put_left(cur, &node->container_node_member);                \
+				next = btree_right(cur);                       \
-					break;                                                            \
+                                                                               \
-				}                                                                         \
+				if (!next) {                                   \
-			} else {                                                                          \
+					btree_put_right(                       \
-				return;                                                                   \
+						cur,                           \
-			}                                                                                 \
+						&node->container_node_member); \
-                                                                                                          \
+					break;                                 \
-			cur = next;                                                                       \
+				}                                              \
-		}                                                                                         \
+			} else if (                                            \
-                                                                                                          \
+				node->container_key_member                     \
-		btree_insert_fixup(tree, &node->container_node_member);                                   \
+				< cur_node->container_key_member) {            \
+				next = btree_left(cur);                        \
+                                                                               \
+				if (!next) {                                   \
+					btree_put_left(                        \
+						cur,                           \
+						&node->container_node_member); \
+					break;                                 \
+				}                                              \
+			} else {                                               \
+				return;                                        \
+			}                                                      \
+                                                                               \
+			cur = next;                                            \
+		}                                                              \
+                                                                               \
+		btree_insert_fixup(tree, &node->container_node_member);        \
 	}
 
 /* defines a node insertion function.
-   this function should be used for trees with complex node keys that cannot be directly compared.
+   this function should be used for trees with complex node keys that cannot be
-   a comparator for your keys must be supplied.
+   directly compared. a comparator for your keys must be supplied.
 
    EXAMPLE:
    if you have a tree node type like this:
 
      struct my_tree_node {
              complex_key_t key;
-	     struct btree_node base;
+             struct btree_node base;
      }
 
-   You would need to define a comparator function or macro with the following signature:
+   You would need to define a comparator function or macro with the following
+   signature:
 
      int my_comparator(struct my_tree_node *a, struct my_tree_node *b);
 
@@ -122,102 +153,136 @@ extern "C" {
      return  0   if a == b
      return  1   if a > b
 
-   You would use the following call to generate an insert function for a tree with this node type:
+   You would use the following call to generate an insert function for a tree
+   with this node type:
 
-     BTREE_DEFINE_INSERT(struct my_tree_node, base, key, my_tree_node_insert, my_comparator);
+     BTREE_DEFINE_INSERT(struct my_tree_node, base, key, my_tree_node_insert,
+   my_comparator);
 
    Which would emit a function defined like:
 
-     static void my_tree_node_insert(struct btree *tree, struct my_tree_node *node);
+     static void my_tree_node_insert(struct btree *tree, struct my_tree_node
+   *node);
 
-   @param node_type your custom tree node type. usually a structure that contains a struct btree_node member.
+   @param node_type your custom tree node type. usually a structure that
-   @param container_node_member the name of the struct btree_node member variable within your custom type.
+   contains a struct btree_node member.
-   @param container_key_member the name of the key member variable within your custom type.
+   @param container_node_member the name of the struct btree_node member
+   variable within your custom type.
+   @param container_key_member the name of the key member variable within your
+   custom type.
    @param function_name the name of the function to generate.
-   @param comparator the name of a comparator function or functional-macro that conforms to the
+   @param comparator the name of a comparator function or functional-macro that
-                     requirements listed above.
+   conforms to the requirements listed above.
 */
-#define BTREE_DEFINE_INSERT(node_type, container_node_member, container_key_member, function_name, comparator) \
+#define BTREE_DEFINE_INSERT(                                                   \
-	void function_name(struct btree *tree, node_type *node)                                                \
+	node_type,                                                             \
-	{                                                                                                      \
+	container_node_member,                                                 \
-		if (!tree->b_root) {                                                                           \
+	container_key_member,                                                  \
-			tree->b_root = &node->container_node_member;                                           \
+	function_name,                                                         \
-			btree_insert_fixup(tree, &node->container_node_member);                                \
+	comparator)                                                            \
-			return;                                                                                \
+	void function_name(struct btree *tree, node_type *node)                \
-		}                                                                                              \
+	{                                                                      \
-                                                                                                               \
+		if (!tree->b_root) {                                           \
-		struct btree_node *cur = tree->b_root;                                                         \
+			tree->b_root = &node->container_node_member;           \
-		while (1) {                                                                                    \
+			btree_insert_fixup(                                    \
-			node_type *cur_node = BTREE_CONTAINER(node_type, container_node_member, cur);          \
+				tree,                                          \
-			struct btree_node *next = NULL;                                                        \
+				&node->container_node_member);                 \
-			int cmp = comparator(node, cur_node);                                                  \
+			return;                                                \
-                                                                                                               \
+		}                                                              \
-			if (cmp == 1) {                                                                        \
+                                                                               \
-				next = btree_right(cur);                                                       \
+		struct btree_node *cur = tree->b_root;                         \
-                                                                                                               \
+		while (1) {                                                    \
-				if (!next) {                                                                   \
+			node_type *cur_node = BTREE_CONTAINER(                 \
-					btree_put_right(cur, &node->container_node_member);                    \
+				node_type,                                     \
-					break;                                                                 \
+				container_node_member,                         \
-				}                                                                              \
+				cur);                                          \
-			} else if (cmp == -1) {                                                                \
+			struct btree_node *next = NULL;                        \
-				next = btree_left(cur);                                                        \
+			int cmp = comparator(node, cur_node);                  \
-                                                                                                               \
+                                                                               \
-				if (!next) {                                                                   \
+			if (cmp == 1) {                                        \
-					btree_put_left(cur, &node->container_node_member);                     \
+				next = btree_right(cur);                       \
-					break;                                                                 \
+                                                                               \
-				}                                                                              \
+				if (!next) {                                   \
-			} else {                                                                               \
+					btree_put_right(                       \
-				return;                                                                        \
+						cur,                           \
-			}                                                                                      \
+						&node->container_node_member); \
-                                                                                                               \
+					break;                                 \
-			cur = next;                                                                            \
+				}                                              \
-		}                                                                                              \
+			} else if (cmp == -1) {                                \
-                                                                                                               \
+				next = btree_left(cur);                        \
-		btree_insert_fixup(tree, &node->container_node_member);                                        \
+                                                                               \
+				if (!next) {                                   \
+					btree_put_left(                        \
+						cur,                           \
+						&node->container_node_member); \
+					break;                                 \
+				}                                              \
+			} else {                                               \
+				return;                                        \
+			}                                                      \
+                                                                               \
+			cur = next;                                            \
+		}                                                              \
+                                                                               \
+		btree_insert_fixup(tree, &node->container_node_member);        \
 	}
 
 /* defines a simple tree search function.
-   this function assumes that your nodes have simple integer keys that can be compared with the usual operators.
+   this function assumes that your nodes have simple integer keys that can be
+   compared with the usual operators.
 
    EXAMPLE:
    if you have a tree node type like this:
 
      struct my_tree_node {
              int key;
-	     struct btree_node base;
+             struct btree_node base;
      }
 
-   You would use the following call to generate a search function for a tree with this node type:
+   You would use the following call to generate a search function for a tree
+   with this node type:
 
-     BTREE_DEFINE_SIMPLE_GET(struct my_tree_node, int, base, key, my_tree_node_get);
+     BTREE_DEFINE_SIMPLE_GET(struct my_tree_node, int, base, key,
+   my_tree_node_get);
 
    Which would emit a function defined like:
 
      static struct my_tree_node *my_tree_node_get(struct btree *tree, int key);
 
-   @param node_type your custom tree node type. usually a structure that contains a struct btree_node member.
+   @param node_type your custom tree node type. usually a structure that
-   @param key_type the type name of the key embedded in your custom tree node type. this type must be
+   contains a struct btree_node member.
-                   compatible with the builtin comparison operators.
+   @param key_type the type name of the key embedded in your custom tree node
-   @param container_node_member the name of the struct btree_node member variable within your custom type.
+   type. this type must be compatible with the builtin comparison operators.
-   @param container_key_member the name of the key member variable within your custom type.
+   @param container_node_member the name of the struct btree_node member
+   variable within your custom type.
+   @param container_key_member the name of the key member variable within your
+   custom type.
    @param function_name the name of the function to generate.
 */
-#define BTREE_DEFINE_SIMPLE_GET(node_type, key_type, container_node_member, container_key_member, function_name) \
+#define BTREE_DEFINE_SIMPLE_GET(                                               \
-	node_type *function_name(struct btree *tree, key_type key)                                               \
+	node_type,                                                             \
-	{                                                                                                        \
+	key_type,                                                              \
-		struct btree_node *cur = tree->b_root;                                                           \
+	container_node_member,                                                 \
-		while (cur) {                                                                                    \
+	container_key_member,                                                  \
-			node_type *cur_node = BTREE_CONTAINER(node_type, container_node_member, cur);            \
+	function_name)                                                         \
-			if (key > cur_node->container_key_member) {                                              \
+	node_type *function_name(struct btree *tree, key_type key)             \
-				cur = btree_right(cur);                                                          \
+	{                                                                      \
-			} else if (key < cur_node->container_key_member) {                                       \
+		struct btree_node *cur = tree->b_root;                         \
-				cur = btree_left(cur);                                                           \
+		while (cur) {                                                  \
-			} else {                                                                                 \
+			node_type *cur_node = BTREE_CONTAINER(                 \
-				return cur_node;                                                                 \
+				node_type,                                     \
-			}                                                                                        \
+				container_node_member,                         \
-		}                                                                                                \
+				cur);                                          \
-	                                                                                                         \
+			if (key > cur_node->container_key_member) {            \
-		return NULL;                                                                                     \
+				cur = btree_right(cur);                        \
+			} else if (key < cur_node->container_key_member) {     \
+				cur = btree_left(cur);                         \
+			} else {                                               \
+				return cur_node;                               \
+			}                                                      \
+		}                                                              \
+                                                                               \
+		return NULL;                                                   \
 	}
 
 /* perform an in-order traversal of a binary tree
@@ -230,7 +295,7 @@ extern "C" {
 
      struct my_tree_node {
              int key;
-	     struct btree_node base;
+             struct btree_node base;
      }
 
    and you want to do something like:
@@ -241,15 +306,23 @@ extern "C" {
 
      btree_foreach (struct my_tree_node, node, &my_tree, base) { ... }
 
-   @param iter_type the type name of the iterator variable. this should be the tree's node type, and shouldn't be a pointer.
+   @param iter_type the type name of the iterator variable. this should be the
+   tree's node type, and shouldn't be a pointer.
    @param iter_name the name of the iterator variable.
    @param tree_name a pointer to the tree to traverse.
-   @param node_member the name of the struct btree_node member variable within the tree node type.
+   @param node_member the name of the struct btree_node member variable within
+   the tree node type.
 */
-#define btree_foreach(iter_type, iter_name, tree_name, node_member) \
+#define btree_foreach(iter_type, iter_name, tree_name, node_member)            \
-	for (iter_type *iter_name = BTREE_CONTAINER(iter_type, node_member, btree_first(tree_name)); \
+	for (iter_type *iter_name = BTREE_CONTAINER(                           \
-	     iter_name; \
+		     iter_type,                                                \
-	     iter_name = BTREE_CONTAINER(iter_type, node_member, btree_next(&((iter_name)->node_member))))
+		     node_member,                                              \
+		     btree_first(tree_name));                                  \
+	     iter_name;                                                        \
+	     iter_name = BTREE_CONTAINER(                                      \
+		     iter_type,                                                \
+		     node_member,                                              \
+		     btree_next(&((iter_name)->node_member))))
 
 /* perform an reverse in-order traversal of a binary tree
 
@@ -261,7 +334,7 @@ extern "C" {
 
      struct my_tree_node {
              int key;
-	     struct btree_node base;
+             struct btree_node base;
      }
 
    and you want to do something like:
@@ -272,35 +345,43 @@ extern "C" {
 
      btree_foreach_r (struct my_tree_node, node, &my_tree, base) { ... }
 
-   @param iter_type the type name of the iterator variable. this should be the tree's node type, and shouldn't be a pointer.
+   @param iter_type the type name of the iterator variable. this should be the
+   tree's node type, and shouldn't be a pointer.
    @param iter_name the name of the iterator variable.
    @param tree_name a pointer to the tree to traverse.
-   @param node_member the name of the struct btree_node member variable within the tree node type.
+   @param node_member the name of the struct btree_node member variable within
+   the tree node type.
 */
-#define btree_foreach_r(iter_type, iter_name, tree_name, node_member) \
+#define btree_foreach_r(iter_type, iter_name, tree_name, node_member)          \
-	for (iter_type *iter_name = BTREE_CONTAINER(iter_type, node_member, btree_last(tree_name)); \
+	for (iter_type *iter_name                                              \
-	     iter_name; \
+	     = BTREE_CONTAINER(iter_type, node_member, btree_last(tree_name)); \
-	     iter_name = BTREE_CONTAINER(iter_type, node_member, btree_prev(&((iter_name)->node_member))))
+	     iter_name;                                                        \
+	     iter_name = BTREE_CONTAINER(                                      \
+		     iter_type,                                                \
+		     node_member,                                              \
+		     btree_prev(&((iter_name)->node_member))))
 
-/* binary tree nodes. this *cannot* be used directly. you need to define a custom node type
+/* binary tree nodes. this *cannot* be used directly. you need to define a
-   that contains a member variable of type struct btree_node.
+   custom node type that contains a member variable of type struct btree_node.
 
-   you would then use the supplied macros to define functions to manipulate your custom binary tree.
+   you would then use the supplied macros to define functions to manipulate your
+   custom binary tree.
 */
 struct btree_node {
 	struct btree_node *b_parent, *b_left, *b_right;
 	unsigned short b_height;
 };
 
-/* binary tree. unlike struct btree_node, you can define variables of type struct btree. */
+/* binary tree. unlike struct btree_node, you can define variables of type
+ * struct btree. */
 struct btree {
 	struct btree_node *b_root;
 };
 
 /* re-balance a binary tree after an insertion operation.
 
-   NOTE that, if you define an insertion function using BTREE_DEFINE_INSERT or similar,
+   NOTE that, if you define an insertion function using BTREE_DEFINE_INSERT or
-   this function will automatically called for you.
+   similar, this function will automatically called for you.
 
    @param tree the tree to re-balance.
    @param node the node that was just inserted into the tree.
@@ -316,29 +397,42 @@ extern void btree_delete(struct btree *tree, struct btree_node *node);
 
 /* get the first node in a binary tree.
 
-   this will be the node with the smallest key (i.e. the node that is furthest-left from the root)
+   this will be the node with the smallest key (i.e. the node that is
+   furthest-left from the root)
 */
 extern struct btree_node *btree_first(struct btree *tree);
 
 /* get the last node in a binary tree.
 
-   this will be the node with the largest key (i.e. the node that is furthest-right from the root)
+   this will be the node with the largest key (i.e. the node that is
+   furthest-right from the root)
 */
 extern struct btree_node *btree_last(struct btree *tree);
-/* for any binary tree node, this function returns the node with the next-largest key value */
+/* for any binary tree node, this function returns the node with the
+ * next-largest key value */
 extern struct btree_node *btree_next(struct btree_node *node);
-/* for any binary tree node, this function returns the node with the next-smallest key value */
+/* for any binary tree node, this function returns the node with the
+ * next-smallest key value */
 extern struct btree_node *btree_prev(struct btree_node *node);
 
+static inline bool btree_empty(const struct btree *tree)
+{
+	return tree->b_root == NULL;
+}
+
 /* sets `child` as the immediate left-child of `parent` */
-static inline void btree_put_left(struct btree_node *parent, struct btree_node *child)
+static inline void btree_put_left(
+	struct btree_node *parent,
+	struct btree_node *child)
 {
 	parent->b_left = child;
 	child->b_parent = parent;
 }
 
 /* sets `child` as the immediate right-child of `parent` */
-static inline void btree_put_right(struct btree_node *parent, struct btree_node *child)
+static inline void btree_put_right(
+	struct btree_node *parent,
+	struct btree_node *child)
 {
 	parent->b_right = child;
 	child->b_parent = parent;

include/kernel/channel.h

@@ -0,0 +1,55 @@
+#ifndef KERNEL_CHANNEL_H_
+#define KERNEL_CHANNEL_H_
+
+#include <kernel/object.h>
+#include <kernel/sched.h>
+
+struct msg;
+
+struct channel {
+	struct object c_base;
+	unsigned int c_id;
+	unsigned int c_msg_waiting;
+	struct btree c_msg;
+	struct btree_node c_node;
+};
+
+extern kern_status_t channel_type_init(void);
+extern struct channel *channel_cast(struct object *obj);
+
+extern struct channel *channel_create(void);
+
+extern kern_status_t channel_enqueue_msg(
+	struct channel *channel,
+	struct msg *msg);
+
+extern kern_status_t channel_recv_msg(
+	struct channel *channel,
+	kern_msg_t *out_msg,
+	unsigned long *irq_flags);
+extern kern_status_t channel_reply_msg(
+	struct channel *channel,
+	msgid_t id,
+	const kern_msg_t *reply,
+	unsigned long *irq_flags);
+
+extern kern_status_t channel_read_msg(
+	struct channel *channel,
+	msgid_t msg,
+	size_t offset,
+	struct address_space *dest_region,
+	const kern_iovec_t *dest_iov,
+	size_t dest_iov_count,
+	size_t *nr_read);
+extern kern_status_t channel_write_msg(
+	struct channel *channel,
+	msgid_t msg,
+	size_t offset,
+	struct address_space *src_region,
+	const kern_iovec_t *src_iov,
+	size_t src_iov_count,
+	size_t *nr_written);
+
+DEFINE_OBJECT_LOCK_FUNCTION(channel, c_base)
+
+#endif

include/kernel/clock.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_CLOCK_H_
+#ifndef KERNEL_CLOCK_H_
-#define MANGO_CLOCK_H_
+#define KERNEL_CLOCK_H_
 
 #include <stdint.h>
 

include/kernel/compiler.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_COMPILER_H_
+#ifndef KERNEL_COMPILER_H_
-#define MANGO_COMPILER_H_
+#define KERNEL_COMPILER_H_
 
 #ifdef __cplusplus
 template <typename T>

include/kernel/console.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_CONSOLE_H_
+#ifndef KERNEL_CONSOLE_H_
-#define MANGO_CONSOLE_H_
+#define KERNEL_CONSOLE_H_
 
 /* The console system
 
@@ -14,9 +14,10 @@
    representing a serial port may allow both sending AND receiving over the
    port.
 */
-#include <mango/queue.h>
+#include <kernel/locks.h>
-#include <mango/locks.h>
+#include <kernel/queue.h>
 #include <mango/status.h>
+#include <mango/types.h>
 
 #ifdef __cplusplus
 extern "C" {

include/kernel/cpu.h

@@ -1,10 +1,11 @@
-#ifndef MANGO_CPU_H_
+#ifndef KERNEL_CPU_H_
-#define MANGO_CPU_H_
+#define KERNEL_CPU_H_
 
-#include <mango/types.h>
+#include <kernel/machine/cpu.h>
-#include <mango/machine/cpu.h>
+#include <kernel/sched.h>
+#include <kernel/types.h>
+#include <kernel/work.h>
 #include <stdint.h>
-#include <mango/sched.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -26,7 +27,7 @@ struct cpu_data {
 
 /* maximum number of processor cores that the kernel can support.
    TODO move to build config option */
-#define CPU_MAX 128
+#define CPU_MAX    128
 
 #define this_cpu() (ml_cpu_block_get_id(ml_this_cpu()))
 
@@ -49,7 +50,7 @@ static inline cycles_t cycles_diff(cycles_t then, cycles_t now)
 	return now >= then ? now - then : (CYCLES_MAX - then) + now;
 }
 
-#define irq_enable() ml_int_enable()
+#define irq_enable()  ml_int_enable()
 #define irq_disable() ml_int_disable()
 
 extern void preempt_disable(void);

include/kernel/equeue.h

@@ -0,0 +1,34 @@
+#ifndef KERNEL_EQUEUE_H_
+#define KERNEL_EQUEUE_H_
+
+#include <kernel/locks.h>
+#include <kernel/object.h>
+#include <kernel/sched.h>
+#include <mango/types.h>
+
+#define EQUEUE_PACKET_MAX 100
+
+enum equeue_flags {
+	EQUEUE_WAIT,
+	EQUEUE_DISCARD,
+};
+
+struct equeue {
+	struct object eq_base;
+	unsigned short eq_read_ptr, eq_write_ptr;
+	equeue_packet_t eq_packets[EQUEUE_PACKET_MAX];
+	struct waitqueue eq_wq;
+};
+
+extern kern_status_t equeue_type_init(void);
+extern struct equeue *equeue_cast(struct object *obj);
+
+extern struct equeue *equeue_create(void);
+
+extern kern_status_t equeue_enqueue(
+	struct equeue *q,
+	const equeue_packet_t *pkt,
+	enum equeue_flags flags);
+extern kern_status_t equeue_dequeue(struct equeue *q, equeue_packet_t *out);
+
+#endif

include/kernel/fb.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_FB_H_
+#ifndef KERNEL_FB_H_
-#define MANGO_FB_H_
+#define KERNEL_FB_H_
 
 #include <stdint.h>
 

include/kernel/flags.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_FLAGS_H_
+#ifndef KERNEL_FLAGS_H_
-#define MANGO_FLAGS_H_
+#define KERNEL_FLAGS_H_
 
 #include <stdint.h>
 

include/kernel/handle.h

@@ -0,0 +1,69 @@
+#ifndef KERNEL_HANDLE_H_
+#define KERNEL_HANDLE_H_
+
+#include <kernel/bitmap.h>
+#include <mango/status.h>
+#include <mango/types.h>
+#include <stddef.h>
+#include <stdint.h>
+
+/* subtract 32 bytes to account for the handle bitmap */
+#define HANDLES_PER_TABLE ((4096 - 32) / sizeof(struct handle))
+#define REFS_PER_TABLE    ((4096 - 64) / sizeof(struct handle_table *))
+
+typedef uint32_t kern_handle_t;
+
+typedef uintptr_t handle_flags_t;
+
+struct task;
+struct object;
+struct address_space;
+struct handle_list;
+
+struct handle {
+	struct object *h_object;
+	handle_flags_t h_flags;
+};
+
+struct handle_table {
+	union {
+		struct {
+			/* bitmap tracks which bits in t_handle_list are
+			 * allocated */
+			DECLARE_BITMAP(t_handle_map, HANDLES_PER_TABLE);
+			struct handle t_handle_list[HANDLES_PER_TABLE];
+		} t_handles;
+
+		struct {
+			/* bitmap tracks which sub-tables are fully-allocated */
+			DECLARE_BITMAP(t_subtable_map, REFS_PER_TABLE);
+			struct handle_table *t_subtable_list[REFS_PER_TABLE];
+		} t_subtables;
+	};
+};
+
+extern struct handle_table *handle_table_create(void);
+extern void handle_table_destroy(struct handle_table *tab);
+
+extern kern_status_t handle_table_alloc_handle(
+	struct handle_table *tab,
+	struct handle **out_slot,
+	kern_handle_t *out_handle);
+extern kern_status_t handle_table_free_handle(
+	struct handle_table *tab,
+	kern_handle_t handle);
+extern struct handle *handle_table_get_handle(
+	struct handle_table *tab,
+	kern_handle_t handle);
+
+extern kern_status_t handle_table_transfer(
+	struct address_space *dst_region,
+	struct handle_table *dst,
+	kern_msg_handle_t *dst_handles,
+	size_t dst_handles_max,
+	struct address_space *src_region,
+	struct handle_table *src,
+	kern_msg_handle_t *src_handles,
+	size_t src_handles_count);
+
+#endif

include/kernel/init.h

@@ -1,8 +1,8 @@
-#ifndef MANGO_INIT_H_
+#ifndef KERNEL_INIT_H_
-#define MANGO_INIT_H_
+#define KERNEL_INIT_H_
 
-#include <mango/compiler.h>
+#include <kernel/compiler.h>
-#include <mango/machine/init.h>
+#include <kernel/machine/init.h>
 
 #ifdef __cplusplus
 extern "C" {

include/kernel/input.h

@@ -1,8 +1,8 @@
-#ifndef MANGO_INPUT_H_
+#ifndef KERNEL_INPUT_H_
-#define MANGO_INPUT_H_
+#define KERNEL_INPUT_H_
 
 #include <stdint.h>
-#include <mango/queue.h>
+#include <kernel/queue.h>
 #include <mango/status.h>
 
 enum input_event_hook_flags {

include/kernel/iovec.h

@@ -0,0 +1,33 @@
+#ifndef KERNEL_IOVEC_H_
+#define KERNEL_IOVEC_H_
+
+#include <mango/types.h>
+#include <stddef.h>
+
+struct address_space;
+
+struct iovec_iterator {
+	/* if this is set, we are iterating over a list of iovecs stored in
+	 * userspace, and must go through this region to retrieve the data. */
+	struct address_space *it_region;
+	const kern_iovec_t *it_vecs;
+	size_t it_nr_vecs;
+	size_t it_vec_ptr;
+
+	virt_addr_t it_base;
+	size_t it_len;
+};
+
+extern void iovec_iterator_begin(
+	struct iovec_iterator *it,
+	const kern_iovec_t *vecs,
+	size_t nr_vecs);
+extern void iovec_iterator_begin_user(
+	struct iovec_iterator *it,
+	struct address_space *address_space,
+	const kern_iovec_t *vecs,
+	size_t nr_vecs);
+
+extern void iovec_iterator_seek(struct iovec_iterator *it, size_t nr_bytes);
+
+#endif

include/kernel/locks.h

@@ -0,0 +1,87 @@
+#ifndef KERNEL_LOCKS_H_
+#define KERNEL_LOCKS_H_
+
+#include <kernel/compiler.h>
+#include <kernel/machine/hwlock.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef __aligned(8) ml_hwlock_t spin_lock_t;
+
+#define SPIN_LOCK_INIT                ML_HWLOCK_INIT
+
+#define spin_lock(lck)                ml_hwlock_lock(lck);
+#define spin_unlock(lck)              ml_hwlock_unlock(lck);
+
+#define spin_lock_irq(lck)            ml_hwlock_lock_irq(lck);
+#define spin_unlock_irq(lck)          ml_hwlock_unlock_irq(lck);
+
+#define spin_lock_irqsave(lck, flags) ml_hwlock_lock_irqsave(lck, flags);
+#define spin_unlock_irqrestore(lck, flags)                                     \
+	ml_hwlock_unlock_irqrestore(lck, flags);
+
+static inline void spin_lock_pair(spin_lock_t *a, spin_lock_t *b)
+{
+	if (a == b) {
+		spin_lock(a);
+	} else if (a < b) {
+		spin_lock(a);
+		spin_lock(b);
+	} else {
+		spin_lock(b);
+		spin_lock(a);
+	}
+}
+
+static inline void spin_unlock_pair(spin_lock_t *a, spin_lock_t *b)
+{
+	if (a == b) {
+		spin_unlock(a);
+	} else if (a < b) {
+		spin_unlock(b);
+		spin_unlock(a);
+	} else {
+		spin_unlock(a);
+		spin_unlock(b);
+	}
+}
+
+static inline void spin_lock_pair_irqsave(
+	spin_lock_t *a,
+	spin_lock_t *b,
+	unsigned long *flags)
+{
+	if (a == b) {
+		spin_lock_irqsave(a, flags);
+	} else if (a < b) {
+		spin_lock_irqsave(a, flags);
+		spin_lock(b);
+	} else {
+		spin_lock_irqsave(b, flags);
+		spin_lock(a);
+	}
+}
+
+static inline void spin_unlock_pair_irqrestore(
+	spin_lock_t *a,
+	spin_lock_t *b,
+	unsigned long flags)
+{
+	if (a == b) {
+		spin_unlock_irqrestore(a, flags);
+	} else if (a < b) {
+		spin_unlock(b);
+		spin_unlock_irqrestore(a, flags);
+	} else {
+		spin_unlock(a);
+		spin_unlock_irqrestore(b, flags);
+	}
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

include/kernel/memblock.h

@@ -19,24 +19,25 @@
     contributors may be used to endorse or promote products derived from this
     software without specific prior written permission.
  */
-#ifndef MANGO_MEMBLOCK_H_
+#ifndef KERNEL_MEMBLOCK_H_
-#define MANGO_MEMBLOCK_H_
+#define KERNEL_MEMBLOCK_H_
 
-#include <stddef.h>
+#include <kernel/types.h>
 #include <limits.h>
-#include <mango/types.h>
+#include <stddef.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#define MEMBLOCK_INIT_MEMORY_REGION_COUNT 128
+#define MEMBLOCK_INIT_MEMORY_REGION_COUNT   128
 #define MEMBLOCK_INIT_RESERVED_REGION_COUNT 128
 
-#define __for_each_mem_range(i, type_a, type_b, p_start, p_end) \
+#define __for_each_mem_range(i, type_a, type_b, p_start, p_end)                \
-	for ((i)->__idx = 0, __next_memory_region(i, type_a, type_b, p_start, p_end); \
+	for ((i)->__idx = 0,                                                   \
-			(i)->__idx != ULLONG_MAX; \
+	    __next_memory_region(i, type_a, type_b, p_start, p_end);           \
-			__next_memory_region(i, type_a, type_b, p_start, p_end))
+	     (i)->__idx != ULLONG_MAX;                                         \
+	     __next_memory_region(i, type_a, type_b, p_start, p_end))
 
 /* iterate through all memory regions known to memblock.
 
@@ -47,7 +48,7 @@ extern "C" {
 
    @param i the iterator. this should be a pointer of type struct memblock_iter.
             for each iteration, this structure will be filled with details about
-	    the current memory region.
+            the current memory region.
    @param p_start the lower bound of the memory region to iterate through.
             if you don't want to use a lower bound, pass 0.
    @param p_end the upper bound of the memory region to iterate through.
@@ -65,7 +66,7 @@ extern "C" {
      struct memblock_iter it;
      for_each_mem_region (&it, 0x40000, 0x80000) { ... }
 */
-#define for_each_mem_range(i, p_start, p_end) \
+#define for_each_mem_range(i, p_start, p_end)                                  \
 	__for_each_mem_range(i, &memblock.memory, NULL, p_start, p_end)
 
 /* iterate through all memory regions reserved using memblock.
@@ -77,7 +78,7 @@ extern "C" {
 
    @param i the iterator. this should be a pointer of type struct memblock_iter.
             for each iteration, this structure will be filled with details about
-	    the current memory region.
+            the current memory region.
    @param p_start the lower bound of the memory region to iterate through.
             if you don't want to use a lower bound, pass 0.
    @param p_end the upper bound of the memory region to iterate through.
@@ -95,7 +96,7 @@ extern "C" {
      struct memblock_iter it;
      for_each_reserved_mem_region (&it, 0x40000, 0x80000) { ... }
 */
-#define for_each_reserved_mem_range(i, p_start, p_end) \
+#define for_each_reserved_mem_range(i, p_start, p_end)                         \
 	__for_each_mem_range(i, &memblock.reserved, NULL, p_start, p_end)
 
 /* iterate through all memory regions known by memblock to be free.
@@ -108,7 +109,7 @@ extern "C" {
 
    @param i the iterator. this should be a pointer of type struct memblock_iter.
             for each iteration, this structure will be filled with details about
-	    the current memory region.
+            the current memory region.
    @param p_start the lower bound of the memory region to iterate through.
             if you don't want to use a lower bound, pass 0.
    @param p_end the upper bound of the memory region to iterate through.
@@ -138,19 +139,25 @@ extern "C" {
      - 0x08000 -> 0x08fff
      - 0x10000 -> 0x1ffff
 */
-#define for_each_free_mem_range(i, p_start, p_end) \
+#define for_each_free_mem_range(i, p_start, p_end)                             \
-	__for_each_mem_range(i, &memblock.memory, &memblock.reserved, p_start, p_end)
+	__for_each_mem_range(                                                  \
+		i,                                                             \
+		&memblock.memory,                                              \
+		&memblock.reserved,                                            \
+		p_start,                                                       \
+		p_end)
 
 typedef uint64_t memblock_index_t;
 
 enum memblock_region_status {
-	/* Used in memblock.memory regions, indicates that the memory region exists */
+	/* Used in memblock.memory regions, indicates that the memory region
+	 * exists */
 	MEMBLOCK_MEMORY = 0,
-	/* Used in memblock.reserved regions, indicates that the memory region was reserved
+	/* Used in memblock.reserved regions, indicates that the memory region
-	 * by a call to memblock_alloc() */
+	 * was reserved by a call to memblock_alloc() */
 	MEMBLOCK_ALLOC,
-	/* Used in memblock.reserved regions, indicates that the memory region was reserved
+	/* Used in memblock.reserved regions, indicates that the memory region
-	 * by a call to memblock_reserve() */
+	 * was reserved by a call to memblock_reserve() */
 	MEMBLOCK_RESERVED,
 };
 
@@ -176,9 +183,10 @@ struct memblock {
 	/* bounds of the memory region that can be used by memblock_alloc()
 	   both of these are virtual addresses */
 	uintptr_t m_alloc_start, m_alloc_end;
-	/* memblock assumes that all memory in the alloc zone is contiguously mapped
+	/* memblock assumes that all memory in the alloc zone is contiguously
-	   (if paging is enabled). m_voffset is the offset that needs to be added to
+	   mapped (if paging is enabled). m_voffset is the offset that needs to
-	   a given physical address to get the corresponding virtual address */
+	   be added to a given physical address to get the corresponding virtual
+	   address */
 	uintptr_t m_voffset;
 
 	struct memblock_type memory;
@@ -212,7 +220,10 @@ extern int __next_mem_range(struct memblock_iter *it);
    @param voffset the offset between the physical address of a given page and
                   its corresponding virtual address.
 */
-extern int memblock_init(uintptr_t alloc_start, uintptr_t alloc_end, uintptr_t voffset);
+extern int memblock_init(
+	uintptr_t alloc_start,
+	uintptr_t alloc_end,
+	uintptr_t voffset);
 
 /* add a region of memory to memblock.
 
@@ -234,7 +245,8 @@ extern int memblock_add(phys_addr_t base, size_t size);
    reserved memory will not be used by memblock_alloc(), and will remain
    reserved when the vm_page memory map is initialised.
 
-   @param base the physical address of the start of the memory region to reserve.
+   @param base the physical address of the start of the memory region to
+   reserve.
    @oaram size the size of the memory region to reserve in bytes.
 */
 extern int memblock_reserve(phys_addr_t base, size_t size);
@@ -257,7 +269,7 @@ extern int memblock_reserve(phys_addr_t base, size_t size);
    @param size the size of the buffer to allocate in bytes.
    @param align the alignment to use. for example, an alignment of 4096
                 will result in the returned pointer being a multiple
-		of 4096. this must be a power of 2.
+                of 4096. this must be a power of 2.
 */
 extern void *memblock_alloc(size_t size, phys_addr_t align);
 
@@ -279,7 +291,7 @@ extern void *memblock_alloc(size_t size, phys_addr_t align);
    @param size the size of the buffer to allocate in bytes.
    @param align the alignment to use. for example, an alignment of 4096
                 will result in the returned pointer being a multiple
-		of 4096. this must be a power of 2.
+                of 4096. this must be a power of 2.
 */
 extern phys_addr_t memblock_alloc_phys(size_t size, phys_addr_t align);
 
@@ -310,7 +322,7 @@ extern int memblock_free_phys(phys_addr_t addr, size_t size);
 
    @param p the pointer to convert.
 */
-extern phys_addr_t memblock_virt_to_phys(void *p);
+extern phys_addr_t memblock_virt_to_phys(const void *p);
 
 /* convert a physical address returned by memblock
    to a virtual pointer.
@@ -319,9 +331,14 @@ extern phys_addr_t memblock_virt_to_phys(void *p);
 */
 extern void *memblock_phys_to_virt(phys_addr_t p);
 
-extern void __next_memory_region(struct memblock_iter *it, \
+extern void __next_memory_region(
-		struct memblock_type *type_a, struct memblock_type *type_b,
+	struct memblock_iter *it,
-		phys_addr_t start, phys_addr_t end);
+	struct memblock_type *type_a,
+	struct memblock_type *type_b,
+	phys_addr_t start,
+	phys_addr_t end);
+
+extern void memblock_dump(void);
 
 #ifdef __cplusplus
 }

include/kernel/msg.h

@@ -0,0 +1,29 @@
+#ifndef KERNEL_MSG_H_
+#define KERNEL_MSG_H_
+
+#include <kernel/btree.h>
+#include <kernel/locks.h>
+#include <mango/status.h>
+#include <mango/types.h>
+
+struct port;
+struct thread;
+
+enum kmsg_status {
+	KMSG_WAIT_RECEIVE,
+	KMSG_WAIT_REPLY,
+	KMSG_REPLY_SENT,
+};
+
+struct msg {
+	spin_lock_t msg_lock;
+	enum kmsg_status msg_status;
+	struct btree_node msg_node;
+	msgid_t msg_id;
+	kern_status_t msg_result;
+	struct port *msg_sender_port;
+	struct thread *msg_sender_thread;
+	kern_msg_t msg_req, msg_resp;
+};
+
+#endif

include/kernel/object.h

@@ -0,0 +1,147 @@
+#ifndef KERNEL_OBJECT_H_
+#define KERNEL_OBJECT_H_
+
+#include <kernel/flags.h>
+#include <kernel/locks.h>
+#include <kernel/vm.h>
+#include <kernel/wait.h>
+#include <mango/status.h>
+#include <stddef.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define DEFINE_OBJECT_LOCK_FUNCTION(object_name, base)                         \
+	static inline void object_name##_lock(struct object_name *p)           \
+	{                                                                      \
+		object_lock(&p->base);                                         \
+	}                                                                      \
+	static inline void object_name##_unlock(struct object_name *p)         \
+	{                                                                      \
+		object_unlock(&p->base);                                       \
+	}                                                                      \
+	static inline void object_name##_lock_irqsave(                         \
+		struct object_name *p,                                         \
+		unsigned long *flags)                                          \
+	{                                                                      \
+		object_lock_irqsave(&p->base, flags);                          \
+	}                                                                      \
+	static inline void object_name##_unlock_irqrestore(                    \
+		struct object_name *p,                                         \
+		unsigned long flags)                                           \
+	{                                                                      \
+		object_unlock_irqrestore(&p->base, flags);                     \
+	}                                                                      \
+	static inline void object_name##_lock_pair(                            \
+		struct object_name *a,                                         \
+		struct object_name *b)                                         \
+	{                                                                      \
+		object_lock_pair(&a->base, &b->base);                          \
+	}                                                                      \
+	static inline void object_name##_unlock_pair(                          \
+		struct object_name *a,                                         \
+		struct object_name *b)                                         \
+	{                                                                      \
+		object_unlock_pair(&a->base, &b->base);                        \
+	}                                                                      \
+	static inline void object_name##_lock_pair_irqsave(                    \
+		struct object_name *a,                                         \
+		struct object_name *b,                                         \
+		unsigned long *flags)                                          \
+	{                                                                      \
+		object_lock_pair_irqsave(&a->base, &b->base, flags);           \
+	}                                                                      \
+	static inline void object_name##_unlock_pair_irqrestore(               \
+		struct object_name *a,                                         \
+		struct object_name *b,                                         \
+		unsigned long flags)                                           \
+	{                                                                      \
+		object_unlock_pair_irqrestore(&a->base, &b->base, flags);      \
+	}
+
+#define OBJECT_MAGIC    0xBADDCAFE
+#define OBJECT_NAME_MAX 64
+#define OBJECT_PATH_MAX 256
+
+#define OBJECT_CAST(to_type, to_type_member, p)                                \
+	((to_type *)((uintptr_t)p) - offsetof(to_type, to_type_member))
+#define OBJECT_C_CAST(c_type, c_type_member, obj_type, objp)                   \
+	OBJECT_IS_TYPE(objp, obj_type)                                         \
+	? OBJECT_CAST(c_type, c_type_member, (objp)) : NULL
+#define OBJECT_IS_TYPE(obj, type_ptr) ((obj)->ob_type == (type_ptr))
+
+struct object;
+struct object_attrib;
+
+enum object_type_flags {
+	OBJTYPE_INIT = 0x01u,
+};
+
+struct object_ops {
+	kern_status_t (*destroy)(struct object *obj, struct queue *q);
+	kern_status_t (*destroy_recurse)(
+		struct queue_entry *entry,
+		struct object **out);
+};
+
+struct object_type {
+	enum object_type_flags ob_flags;
+	char ob_name[32];
+	unsigned int ob_size;
+	unsigned int ob_header_offset;
+	struct vm_cache ob_cache;
+	struct queue_entry ob_list;
+	struct object_ops ob_ops;
+};
+
+struct object {
+	uint32_t ob_magic;
+	koid_t ob_id;
+	struct object_type *ob_type;
+	spin_lock_t ob_lock;
+	uint32_t ob_signals;
+	unsigned int ob_refcount;
+	unsigned int ob_handles;
+	struct queue_entry ob_list;
+	struct waitqueue ob_wq;
+} __aligned(sizeof(long));
+
+extern kern_status_t object_bootstrap(void);
+extern kern_status_t object_type_register(struct object_type *p);
+extern kern_status_t object_type_unregister(struct object_type *p);
+
+extern struct object *object_create(struct object_type *type);
+extern struct object *object_ref(struct object *obj);
+extern void object_unref(struct object *obj);
+extern void object_add_handle(struct object *obj);
+extern void object_remove_handle(struct object *obj);
+extern void object_lock(struct object *obj);
+extern void object_unlock(struct object *obj);
+extern void object_lock_irqsave(struct object *obj, unsigned long *flags);
+extern void object_unlock_irqrestore(struct object *obj, unsigned long flags);
+
+extern void object_lock_pair(struct object *a, struct object *b);
+extern void object_unlock_pair(struct object *a, struct object *b);
+
+extern void object_lock_pair_irqsave(
+	struct object *a,
+	struct object *b,
+	unsigned long *flags);
+extern void object_unlock_pair_irqrestore(
+	struct object *a,
+	struct object *b,
+	unsigned long flags);
+
+extern void object_assert_signal(struct object *obj, uint32_t signals);
+extern void object_clear_signal(struct object *obj, uint32_t signals);
+extern void object_wait_signal(
+	struct object *obj,
+	uint32_t signals,
+	unsigned long *irq_flags);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

include/kernel/panic.h

@@ -0,0 +1,13 @@
+#ifndef KERNEL_PANIC_H_
+#define KERNEL_PANIC_H_
+
+#include <kernel/compiler.h>
+
+struct ml_cpu_context;
+
+#define panic(...) panic_irq(NULL, __VA_ARGS__)
+
+extern void __noreturn
+panic_irq(struct ml_cpu_context *ctx, const char *fmt, ...);
+
+#endif

include/kernel/percpu.h

@@ -1,9 +1,9 @@
-#ifndef MANGO_PERCPU_H_
+#ifndef KERNEL_PERCPU_H_
-#define MANGO_PERCPU_H_
+#define KERNEL_PERCPU_H_
 
 #include <mango/status.h>
-#include <mango/compiler.h>
+#include <kernel/compiler.h>
-#include <mango/sched.h>
+#include <kernel/sched.h>
 
 #ifdef __cplusplus
 extern "C" {

include/kernel/pmap.h

@@ -0,0 +1,81 @@
+#ifndef KERNEL_PMAP_H_
+#define KERNEL_PMAP_H_
+
+/* all the functions declared in this file are defined in arch/xyz/pmap.c */
+
+#include <kernel/machine/pmap.h>
+#include <kernel/vm.h>
+#include <mango/status.h>
+#include <stddef.h>
+
+#define PMAP_INVALID ML_PMAP_INVALID
+#define PFN(x)       ((x) >> VM_PAGE_SHIFT)
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef ml_pmap_t pmap_t;
+typedef ml_pfn_t pfn_t;
+
+enum pmap_fault_flags {
+	/* if set, the faulting page is present, and the fault is
+	 * protection-related.
+	 * if clear, the faulting page is missing, and the
+	 * fault is due to the missing page.
+	 */
+	PMAP_FAULT_PRESENT = 0x01u,
+	/* if set, the faulting page was accessed from user mode.
+	 * if clear, the faulting page was accessed from kernel mode.
+	 */
+	PMAP_FAULT_USER = 0x02u,
+	/* if set, the fault was caused by a write operation.
+	 * if clear, the faulting page was caused by a read operation.
+	 */
+	PMAP_FAULT_WRITE = 0x04u,
+	/* if set, the fault was caused while fetching an instruction from the
+	 * faulting page.
+	 */
+	PMAP_FAULT_IFETCH = 0x08u,
+	/* if set, the fault was caused by misconfigured page tables */
+	PMAP_FAULT_BADCFG = 0x10u,
+};
+
+enum pmap_flags {
+	PMAP_NORMAL = 0x00u,
+	PMAP_HUGEPAGE = 0x01u,
+};
+
+extern void pmap_bootstrap(void);
+extern pmap_t get_kernel_pmap(void);
+
+extern pmap_t pmap_create(void);
+extern void pmap_destroy(pmap_t pmap);
+extern void pmap_switch(pmap_t pmap);
+
+extern kern_status_t pmap_handle_fault(
+	virt_addr_t fault_addr,
+	enum pmap_fault_flags flags);
+
+extern kern_status_t pmap_add(
+	pmap_t pmap,
+	virt_addr_t p,
+	pfn_t pfn,
+	vm_prot_t prot,
+	enum pmap_flags flags);
+extern kern_status_t pmap_add_block(
+	pmap_t pmap,
+	virt_addr_t p,
+	pfn_t pfn,
+	size_t len,
+	vm_prot_t prot,
+	enum pmap_flags flags);
+
+extern kern_status_t pmap_remove(pmap_t pmap, virt_addr_t p);
+extern kern_status_t pmap_remove_range(pmap_t pmap, virt_addr_t p, size_t len);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

include/kernel/port.h

@@ -0,0 +1,41 @@
+#ifndef KERNEL_PORT_H_
+#define KERNEL_PORT_H_
+
+#include <kernel/object.h>
+#include <kernel/sched.h>
+
+enum port_status {
+	/* port is not connected */
+	PORT_OFFLINE = 0,
+	/* port is connected and ready to send messages */
+	PORT_READY,
+	/* port has sent a message, and is waiting for the remote to receive it
+	 */
+	PORT_SEND_BLOCKED,
+	/* port has sent a message, and the remote has received it. waiting for
+	 * the remote to reply. */
+	PORT_REPLY_BLOCKED,
+};
+
+struct port {
+	struct object p_base;
+	enum port_status p_status;
+	struct channel *p_remote;
+};
+
+extern kern_status_t port_type_init(void);
+extern struct port *port_cast(struct object *obj);
+
+extern struct port *port_create(void);
+
+extern kern_status_t port_connect(struct port *port, struct channel *remote);
+extern kern_status_t port_disconnect(struct port *port);
+extern kern_status_t port_send_msg(
+	struct port *port,
+	const kern_msg_t *msg,
+	kern_msg_t *out_response,
+	unsigned long *lock_flags);
+
+DEFINE_OBJECT_LOCK_FUNCTION(port, p_base)
+
+#endif

include/kernel/printk.h

@@ -0,0 +1,25 @@
+#ifndef KERNEL_PRINTK_H_
+#define KERNEL_PRINTK_H_
+
+#include <kernel/console.h>
+
+#undef TRACE
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef TRACE
+#define tracek(...) printk(__VA_ARGS__)
+#else
+#define tracek(...)
+#endif
+
+extern void early_printk_init(struct console *con);
+extern int printk(const char *format, ...);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

include/kernel/queue.h

@@ -1,7 +1,7 @@
-#ifndef MANGO_QUEUE_H_
+#ifndef KERNEL_QUEUE_H_
-#define MANGO_QUEUE_H_
+#define KERNEL_QUEUE_H_
 
-#include <mango/libc/string.h>
+#include <kernel/libc/string.h>
 #include <stdbool.h>
 
 #ifdef __cplusplus

include/kernel/ringbuffer.h

@@ -1,8 +1,10 @@
-#ifndef MANGO_RINGBUFFER_H_
+#ifndef KERNEL_RINGBUFFER_H_
-#define MANGO_RINGBUFFER_H_
+#define KERNEL_RINGBUFFER_H_
 
-#include <mango/locks.h>
+#include <kernel/flags.h>
-#include <mango/sched.h>
+#include <kernel/locks.h>
+#include <kernel/types.h>
+#include <kernel/wait.h>
 
 struct ringbuffer {
 	unsigned char *r_buffer;
@@ -22,12 +24,21 @@ extern kern_status_t ringbuffer_deinit(struct ringbuffer *buf);
 
 extern size_t ringbuffer_unread(struct ringbuffer *buf);
 extern size_t ringbuffer_avail(struct ringbuffer *buf);
-extern size_t ringbuffer_read(struct ringbuffer *buf, size_t size, void *buffer, mango_flags_t flags);
+extern size_t ringbuffer_read(
-extern size_t ringbuffer_write(struct ringbuffer *buf, size_t size, const void *buffer, mango_flags_t flags);
+	struct ringbuffer *buf,
+	size_t size,
+	void *buffer,
+	mango_flags_t flags);
+extern size_t ringbuffer_write(
+	struct ringbuffer *buf,
+	size_t size,
+	const void *buffer,
+	mango_flags_t flags);
 
 /* TODO */
-//extern size_t ringbuffer_peek(struct ringbuffer *buf, size_t at, size_t size, void *buffer);
+// extern size_t ringbuffer_peek(struct ringbuffer *buf, size_t at, size_t size,
-//extern size_t ringbuffer_skip(struct ringbuffer *buf, size_t count);
+// void *buffer); extern size_t ringbuffer_skip(struct ringbuffer *buf, size_t
+// count);
 
 extern int ringbuffer_write_would_block(struct ringbuffer *buf);
 

include/kernel/sched.h

@@ -0,0 +1,98 @@
+#ifndef KERNEL_SCHED_H_
+#define KERNEL_SCHED_H_
+
+#include <kernel/btree.h>
+#include <kernel/handle.h>
+#include <kernel/locks.h>
+#include <kernel/queue.h>
+#include <kernel/types.h>
+#include <mango/status.h>
+
+#define PRIO_MAX 32
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct channel;
+struct runqueue;
+
+enum sched_priority {
+	PRIO_IDLE = 4,
+	PRIO_SUBNORMAL = 6,
+	PRIO_NORMAL = 10,
+	PRIO_SUPERNORMAL = 14,
+	PRIO_HIGH = 18,
+	PRIO_REALTIME = 24,
+};
+
+enum sched_mode {
+	/* used when calling from non-interrupt context.
+	   threads that aren't in state THREAD_READY are
+	   removed from the runqueue. */
+	SCHED_NORMAL = 0,
+	/* used when calling from interrupt context.
+	   threads that aren't in state THREAD_READY are
+	   still added to the runqueue. */
+	SCHED_IRQ = 1,
+};
+
+struct runqueue {
+	struct queue rq_queues[PRIO_MAX];
+	uint32_t rq_readybits;
+	spin_lock_t rq_lock;
+	unsigned int rq_nthreads;
+
+	struct thread *rq_cur, *rq_idle;
+};
+
+struct timer {
+	struct queue_entry t_entry;
+	struct cpu_data *t_cpu;
+	struct thread *t_owner;
+	unsigned long t_expiry;
+	void (*t_callback)(struct timer *);
+};
+
+extern kern_status_t sched_init(void);
+extern void schedule(enum sched_mode mode);
+extern void preempt_disable(void);
+extern void preempt_enable(void);
+
+extern void rq_init(struct runqueue *rq);
+extern struct thread *rq_dequeue(struct runqueue *rq);
+extern void rq_enqueue(struct runqueue *rq, struct thread *thr);
+static inline void rq_lock(struct runqueue *rq, unsigned long *flags)
+{
+	spin_lock_irqsave(&rq->rq_lock, flags);
+}
+static inline void rq_unlock(struct runqueue *rq, unsigned long flags)
+{
+	spin_unlock_irqrestore(&rq->rq_lock, flags);
+}
+extern void rq_remove_thread(struct runqueue *rq, struct thread *thr);
+extern struct runqueue *cpu_rq(unsigned int cpu);
+
+extern cycles_t default_quantum(void);
+
+extern bool need_resched(void);
+extern struct task *current_task(void);
+extern struct thread *current_thread(void);
+
+extern struct runqueue *select_rq_for_thread(struct thread *thr);
+extern void schedule_thread_on_cpu(struct thread *thr);
+
+extern void start_charge_period(void);
+extern void end_charge_period(void);
+
+extern void add_timer(struct timer *timer);
+extern void remove_timer(struct timer *timer);
+extern unsigned long schedule_timeout(unsigned long clock_ticks);
+extern unsigned long milli_sleep(unsigned long ms);
+extern void sleep_forever(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

include/kernel/syscall.h

@@ -0,0 +1,202 @@
+#ifndef KERNEL_SYSCALL_H_
+#define KERNEL_SYSCALL_H_
+
+#include <kernel/address-space.h>
+#include <kernel/handle.h>
+#include <kernel/task.h>
+#include <kernel/vm.h>
+#include <mango/status.h>
+#include <mango/syscall.h>
+
+#define validate_access(task, ptr, len, flags)                                 \
+	__validate_access(task, (const void *)ptr, len, flags)
+#define validate_access_r(task, ptr, len)                                      \
+	validate_access(task, ptr, len, VM_PROT_READ | VM_PROT_USER)
+#define validate_access_w(task, ptr, len)                                      \
+	validate_access(task, ptr, len, VM_PROT_WRITE | VM_PROT_USER)
+#define validate_access_rw(task, ptr, len)                                     \
+	validate_access(                                                       \
+		task,                                                          \
+		ptr,                                                           \
+		len,                                                           \
+		VM_PROT_READ | VM_PROT_WRITE | VM_PROT_USER)
+
+static inline bool __validate_access(
+	struct task *task,
+	const void *ptr,
+	size_t len,
+	vm_prot_t flags)
+{
+	unsigned long irq_flags;
+	address_space_lock_irqsave(task->t_address_space, &irq_flags);
+	bool result = address_space_validate_access(
+		task->t_address_space,
+		(virt_addr_t)ptr,
+		len,
+		flags | VM_PROT_USER);
+	address_space_unlock_irqrestore(task->t_address_space, irq_flags);
+	return result;
+}
+
+extern kern_status_t sys_task_exit(int status);
+extern kern_status_t sys_task_self(kern_handle_t *out);
+extern kern_status_t sys_task_create(
+	kern_handle_t parent_handle,
+	const char *name,
+	size_t name_len,
+	kern_handle_t *out_task,
+	kern_handle_t *out_address_space);
+extern kern_status_t sys_task_create_thread(
+	kern_handle_t task,
+	virt_addr_t ip,
+	virt_addr_t sp,
+	uintptr_t *args,
+	size_t nr_args,
+	kern_handle_t *out_thread);
+extern kern_status_t sys_task_get_address_space(
+	kern_handle_t task,
+	kern_handle_t *out);
+
+extern kern_status_t sys_thread_start(kern_handle_t thread);
+
+extern kern_status_t sys_vm_object_create(
+	const char *name,
+	size_t name_len,
+	size_t data_len,
+	vm_prot_t prot,
+	kern_handle_t *out);
+extern kern_status_t sys_vm_object_read(
+	kern_handle_t object,
+	void *dst,
+	off_t offset,
+	size_t count,
+	size_t *nr_read);
+extern kern_status_t sys_vm_object_write(
+	kern_handle_t object,
+	const void *src,
+	off_t offset,
+	size_t count,
+	size_t *nr_written);
+extern kern_status_t sys_vm_object_copy(
+	kern_handle_t dst,
+	off_t dst_offset,
+	kern_handle_t src,
+	off_t src_offset,
+	size_t count,
+	size_t *nr_copied);
+
+extern kern_status_t sys_address_space_read(
+	kern_handle_t region,
+	void *dst,
+	virt_addr_t base,
+	size_t count,
+	size_t *nr_read);
+extern kern_status_t sys_address_space_write(
+	kern_handle_t region,
+	const void *src,
+	virt_addr_t base,
+	size_t count,
+	size_t *nr_read);
+extern kern_status_t sys_address_space_map(
+	kern_handle_t region,
+	virt_addr_t map_address,
+	kern_handle_t object,
+	off_t object_offset,
+	size_t length,
+	vm_prot_t prot,
+	virt_addr_t *out_base_address);
+extern kern_status_t sys_address_space_unmap(
+	kern_handle_t region,
+	virt_addr_t base,
+	size_t length);
+extern kern_status_t sys_address_space_reserve(
+	kern_handle_t region,
+	virt_addr_t base,
+	size_t length,
+	virt_addr_t *out_base_address);
+extern kern_status_t sys_address_space_release(
+	kern_handle_t region,
+	virt_addr_t base,
+	size_t length);
+
+extern kern_status_t sys_kern_log(const char *s);
+extern kern_status_t sys_kern_handle_close(kern_handle_t handle);
+extern kern_status_t sys_kern_handle_duplicate(
+	kern_handle_t handle,
+	kern_handle_t *out);
+extern kern_status_t sys_kern_config_get(
+	kern_config_key_t key,
+	void *ptr,
+	size_t len);
+extern kern_status_t sys_kern_config_set(
+	kern_config_key_t key,
+	const void *ptr,
+	size_t len);
+
+extern kern_status_t sys_channel_create(unsigned int id, kern_handle_t *out);
+extern kern_status_t sys_port_create(kern_handle_t *out);
+extern kern_status_t sys_port_connect(
+	kern_handle_t port,
+	tid_t task_id,
+	unsigned int channel_id);
+extern kern_status_t sys_port_disconnect(kern_handle_t port);
+
+extern kern_status_t sys_msg_send(
+	kern_handle_t port,
+	const kern_msg_t *msg,
+	kern_msg_t *out_reply);
+extern kern_status_t sys_msg_recv(kern_handle_t channel, kern_msg_t *out_msg);
+
+extern kern_status_t sys_msg_reply(
+	kern_handle_t channel,
+	msgid_t id,
+	const kern_msg_t *msg);
+extern kern_status_t sys_msg_read(
+	kern_handle_t channel_handle,
+	msgid_t id,
+	size_t offset,
+	const kern_iovec_t *iov,
+	size_t iov_count,
+	size_t *nr_read);
+extern kern_status_t sys_msg_write(
+	kern_handle_t channel,
+	msgid_t id,
+	size_t offset,
+	const kern_iovec_t *in,
+	size_t nr_in,
+	size_t *nr_written);
+
+extern kern_status_t sys_kern_object_wait(
+	kern_wait_item_t *items,
+	size_t nr_items);
+
+extern kern_status_t sys_vm_controller_create(kern_handle_t *out);
+extern kern_status_t sys_vm_controller_recv(
+	kern_handle_t ctrl,
+	equeue_packet_page_request_t *out);
+extern kern_status_t sys_vm_controller_recv_async(
+	kern_handle_t ctrl,
+	kern_handle_t eq,
+	equeue_key_t key);
+extern kern_status_t sys_vm_controller_create_object(
+	kern_handle_t ctrl,
+	const char *name,
+	size_t name_len,
+	equeue_key_t key,
+	size_t data_len,
+	vm_prot_t prot,
+	kern_handle_t *out);
+extern kern_status_t sys_vm_controller_detach_object(
+	kern_handle_t ctrl,
+	kern_handle_t vmo);
+extern kern_status_t sys_vm_controller_supply_pages(
+	kern_handle_t ctrl,
+	kern_handle_t dst_vmo,
+	off_t dst_offset,
+	kern_handle_t src_vmo,
+	off_t src_offset,
+	size_t count);
+
+extern virt_addr_t syscall_get_function(unsigned int sysid);
+
+#endif

include/kernel/task.h

@@ -0,0 +1,75 @@
+#ifndef KERNEL_TASK_H_
+#define KERNEL_TASK_H_
+
+#include <kernel/handle.h>
+#include <kernel/object.h>
+#include <kernel/pmap.h>
+
+#define TASK_NAME_MAX 64
+#define PID_MAX       99999
+
+struct channel;
+
+enum task_state {
+	TASK_RUNNING,
+	TASK_STOPPED,
+};
+
+struct task {
+	struct object t_base;
+
+	struct task *t_parent;
+	long t_id;
+	enum task_state t_state;
+	char t_name[TASK_NAME_MAX];
+
+	pmap_t t_pmap;
+	struct address_space *t_address_space;
+	spin_lock_t t_handles_lock;
+	struct handle_table *t_handles;
+	struct btree b_channels;
+
+	struct btree_node t_tasklist;
+	struct queue_entry t_child_entry;
+
+	size_t t_next_thread_id;
+	struct queue t_threads;
+	struct queue t_children;
+};
+
+extern struct task *task_alloc(void);
+extern struct task *task_cast(struct object *obj);
+extern struct task *task_create(const char *name, size_t name_len);
+static inline struct task *task_ref(struct task *task)
+{
+	return OBJECT_CAST(struct task, t_base, object_ref(&task->t_base));
+}
+static inline void task_unref(struct task *task)
+{
+	object_unref(&task->t_base);
+}
+extern kern_status_t task_add_child(struct task *parent, struct task *child);
+extern kern_status_t task_add_channel(
+	struct task *task,
+	struct channel *channel,
+	unsigned int id);
+extern struct channel *task_get_channel(struct task *task, unsigned int id);
+extern struct task *task_from_tid(tid_t id);
+extern kern_status_t task_open_handle(
+	struct task *task,
+	struct object *obj,
+	handle_flags_t flags,
+	kern_handle_t *out);
+extern kern_status_t task_resolve_handle(
+	struct task *task,
+	kern_handle_t handle,
+	struct object **out_obj,
+	handle_flags_t *out_flags);
+extern kern_status_t task_close_handle(struct task *task, kern_handle_t handle);
+extern struct thread *task_create_thread(struct task *parent);
+extern struct task *kernel_task(void);
+extern struct task *idle_task(void);
+
+DEFINE_OBJECT_LOCK_FUNCTION(task, t_base)
+
+#endif

include/kernel/test.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_TEST_H_
+#ifndef KERNEL_TEST_H_
-#define MANGO_TEST_H_
+#define KERNEL_TEST_H_
 
 #ifdef __cplusplus
 extern "C" {

include/kernel/thread.h

@@ -0,0 +1,67 @@
+#ifndef KERNEL_THREAD_H_
+#define KERNEL_THREAD_H_
+
+#include <kernel/msg.h>
+#include <kernel/object.h>
+#include <kernel/vm-controller.h>
+
+#define THREAD_KSTACK_ORDER VM_PAGE_4K
+
+enum thread_state {
+	THREAD_READY = 1,
+	THREAD_SLEEPING = 2,
+	THREAD_STOPPED = 3,
+};
+
+enum thread_flags {
+	/* this thread has exhausted its quantum and is due to be re-scheduled.
+	 */
+	THREAD_F_NEED_RESCHED = 0x01u,
+	/* this thread is currently scheduled (i.e. is present on a runqueue) */
+	THREAD_F_SCHEDULED = 0x04u,
+};
+
+struct thread {
+	struct object thr_base;
+
+	enum thread_state tr_state;
+	enum thread_flags tr_flags;
+	struct task *tr_parent;
+
+	unsigned int tr_id;
+	unsigned int tr_prio;
+
+	cycles_t tr_charge_period_start;
+	cycles_t tr_quantum_cycles, tr_quantum_target;
+	cycles_t tr_total_cycles;
+
+	virt_addr_t tr_ip, tr_sp, tr_bp;
+	virt_addr_t tr_cpu_user_sp, tr_cpu_kernel_sp;
+
+	struct runqueue *tr_rq;
+	struct msg tr_msg;
+	struct page_request tr_page_req;
+
+	struct queue_entry tr_parent_entry;
+	struct queue_entry tr_rqentry;
+
+	struct vm_page *tr_kstack;
+	struct vm_object *tr_ustack;
+};
+
+extern struct thread *thread_alloc(void);
+extern struct thread *thread_cast(struct object *obj);
+extern kern_status_t thread_init_kernel(struct thread *thr, virt_addr_t ip);
+extern kern_status_t thread_init_user(
+	struct thread *thr,
+	virt_addr_t ip,
+	virt_addr_t sp,
+	const uintptr_t *args,
+	size_t nr_args);
+extern int thread_priority(struct thread *thr);
+extern void thread_awaken(struct thread *thr);
+extern void idle(void);
+extern struct thread *create_kernel_thread(void (*fn)(void));
+extern struct thread *create_idle_thread(void);
+
+#endif

include/kernel/types.h

@@ -0,0 +1,18 @@
+#ifndef KERNEL_TYPES_H_
+#define KERNEL_TYPES_H_
+
+#include <mango/types.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#define CYCLES_MAX UINT64_MAX
+
+typedef uint64_t cycles_t;
+typedef uint64_t sectors_t;
+
+struct boot_module {
+	phys_addr_t mod_base;
+	size_t mod_size;
+};
+
+#endif

include/kernel/util.h

@@ -1,27 +1,35 @@
-#ifndef MANGO_UTIL_H_
+#ifndef KERNEL_UTIL_H_
-#define MANGO_UTIL_H_
+#define KERNEL_UTIL_H_
 
+#include <mango/types.h>
+#include <stdbool.h>
 #include <stddef.h>
 #include <stdint.h>
-#include <stdbool.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#define MIN(x, y) ((x) < (y) ? (x) : (y))
+#define MIN(x, y)        ((x) < (y) ? (x) : (y))
-#define MAX(x, y) ((x) > (y) ? (x) : (y))
+#define MAX(x, y)        ((x) > (y) ? (x) : (y))
 #define CLAMP(x, lo, hi) (MIN(MAX(x, lo), hi))
 
 extern uint64_t hash_string(const char *s);
 extern void data_size_to_string(size_t value, char *out, size_t outsz);
-static inline bool power_of_2(size_t x) { return (x > 0 && (x & (x - 1)) == 0); }
+static inline bool power_of_2(size_t x)
-static inline unsigned long long div64_pow2(unsigned long long x, unsigned long long y)
+{
+	return (x > 0 && (x & (x - 1)) == 0);
+}
+static inline unsigned long long div64_pow2(
+	unsigned long long x,
+	unsigned long long y)
 {
 	return x >> (__builtin_ctz(y));
 }
 
-static inline unsigned long long absdiff64(unsigned long long x, unsigned long long y)
+static inline unsigned long long absdiff64(
+	unsigned long long x,
+	unsigned long long y)
 {
 	return x < y ? y - x : x - y;
 }
@@ -53,6 +61,9 @@ extern uint64_t host_to_little_u64(uint64_t v);
 extern uint64_t big_to_host_u64(uint64_t v);
 extern uint64_t little_to_host_u64(uint64_t v);
 
+extern void init_random(uint64_t seed);
+extern bool fill_random(void *buffer, unsigned int size);
+
 #ifdef __cplusplus
 }
 #endif

include/kernel/vm-controller.h

@@ -0,0 +1,85 @@
+#ifndef KERNEL_VM_CONTROLLER_H_
+#define KERNEL_VM_CONTROLLER_H_
+
+#include <kernel/locks.h>
+#include <kernel/object.h>
+#include <mango/types.h>
+
+struct thread;
+struct equeue;
+struct vm_object;
+
+enum page_request_status {
+	PAGE_REQUEST_PENDING = 0,
+	PAGE_REQUEST_IN_PROGRESS,
+	PAGE_REQUEST_COMPLETE,
+};
+
+struct vm_controller {
+	struct object vc_base;
+	/* tree of struct vm_objects bound to this controller, keyed with the
+	 * equeue_key_t specified when the object(s) were created. */
+	struct btree vc_objects;
+	/* tree of pending page requests */
+	struct btree vc_requests;
+	/* the equeue to send async page requests to */
+	struct equeue *vc_eq;
+	equeue_key_t vc_eq_key;
+	/* the number of page requests queued with status PAGE_REQUEST_PENDING.
+	 * used to assert/clear VM_CONTROLLER_SIGNAL_REQUEST_RECEIVED */
+	size_t vc_requests_waiting;
+};
+
+struct page_request {
+	uint64_t req_id;
+	unsigned int req_type;
+	enum page_request_status req_status;
+	kern_status_t req_result;
+	spin_lock_t req_lock;
+	struct vm_object *req_object;
+	struct thread *req_sender;
+	struct btree_node req_node;
+	off_t req_offset;
+	size_t req_length;
+};
+
+extern kern_status_t vm_controller_type_init(void);
+extern struct vm_controller *vm_controller_cast(struct object *obj);
+
+extern struct vm_controller *vm_controller_create(void);
+
+extern kern_status_t vm_controller_recv(
+	struct vm_controller *ctrl,
+	equeue_packet_page_request_t *out);
+extern kern_status_t vm_controller_recv_async(
+	struct vm_controller *ctrl,
+	struct equeue *eq,
+	equeue_key_t key);
+
+extern kern_status_t vm_controller_create_object(
+	struct vm_controller *ctrl,
+	const char *name,
+	size_t name_len,
+	equeue_key_t key,
+	size_t data_len,
+	vm_prot_t prot,
+	struct vm_object **out);
+extern kern_status_t vm_controller_detach_object(
+	struct vm_controller *ctrl,
+	struct vm_object *vmo);
+extern kern_status_t vm_controller_supply_pages(
+	struct vm_controller *ctrl,
+	struct vm_object *dst,
+	off_t dst_offset,
+	struct vm_object *src,
+	off_t src_offset,
+	size_t count);
+
+extern kern_status_t vm_controller_send_request(
+	struct vm_controller *ctrl,
+	struct page_request *req,
+	unsigned long *irq_flags);
+
+DEFINE_OBJECT_LOCK_FUNCTION(vm_controller, vc_base)
+
+#endif

include/kernel/vm-object.h

@@ -0,0 +1,116 @@
+#ifndef KERNEL_VM_OBJECT_H_
+#define KERNEL_VM_OBJECT_H_
+
+#include <kernel/locks.h>
+#include <kernel/object.h>
+
+#define VM_OBJECT_NAME_MAX 64
+
+struct vm_controller;
+
+enum vm_object_flags {
+	/* the memory behind this vm-object wasn't allocated by us, and
+	 * therefore shouldn't be freed by us */
+	VMO_IN_PLACE = 0x01u,
+	/* this vm-object is/was attached to a vm-controller */
+	VMO_CONTROLLER = 0x02u,
+
+	/* these flags are for use with vm_object_get_page */
+	/**************************************************/
+
+	/* if the relevant page hasn't been allocated yet, it will be allocated
+	 * and returned. if this flag isn't specified, NULL will be returned. */
+	VMO_ALLOCATE_MISSING_PAGE = 0x04u,
+	/* if the vm-object is attached to a vm-controller, and the relevant
+	 * page is uncommitted, send a request to the vm-controller to provide
+	 * the missing page. will result in the vm-object being unlocked and
+	 * the current thread sleeping until the request is fulfilled. the
+	 * vm-object will be re-locked before the function returns. */
+	VMO_REQUEST_MISSING_PAGE = 0x08u,
+};
+
+struct vm_object {
+	struct object vo_base;
+
+	char vo_name[VM_OBJECT_NAME_MAX];
+	enum vm_object_flags vo_flags;
+
+	/* queue of struct vm_region_mapping */
+	struct queue vo_mappings;
+
+	struct vm_controller *vo_ctrl;
+	equeue_key_t vo_key;
+	struct btree_node vo_ctrl_node;
+
+	/* memory protection flags. mappings of this vm_object can only
+	 * use a subset of the flags set in this mask. */
+	vm_prot_t vo_prot;
+
+	/* btree of vm_pages that have been allocated to this vm_object.
+	 * vm_page->p_vmo_offset and the size of each page is the bst key. */
+	struct btree vo_pages;
+	/* total length of the vm_object in bytes. */
+	size_t vo_size;
+};
+
+extern kern_status_t vm_object_type_init(void);
+extern struct vm_object *vm_object_cast(struct object *obj);
+
+/* create a vm_object with the specified length in bytes and protection flags.
+ * the length will be automatically rounded up to the nearest vm_object page
+ * order size. the actual page frames themselves won't be allocated until
+ * they are mapped and accessed. */
+extern struct vm_object *vm_object_create(
+	const char *name,
+	size_t name_len,
+	size_t data_len,
+	vm_prot_t prot);
+
+/* create a vm_object that represents the specified range of physical memory.
+ * the length will be automatically rounded up to the nearest vm_object page
+ * order size.
+ * NOTE this function assumes that the physical memory has already been
+ * reserved, and is not in use by any other kernel component. */
+extern struct vm_object *vm_object_create_in_place(
+	const char *name,
+	size_t name_len,
+	phys_addr_t base,
+	size_t data_len,
+	vm_prot_t prot);
+
+extern struct vm_page *vm_object_get_page(
+	struct vm_object *vo,
+	off_t offset,
+	enum vm_object_flags flags,
+	unsigned long *irq_flags);
+
+extern kern_status_t vm_object_read(
+	struct vm_object *vo,
+	void *dst,
+	off_t offset,
+	size_t count,
+	size_t *nr_read);
+extern kern_status_t vm_object_write(
+	struct vm_object *vo,
+	const void *dst,
+	off_t offset,
+	size_t count,
+	size_t *nr_written);
+extern kern_status_t vm_object_copy(
+	struct vm_object *dst,
+	off_t dst_offset,
+	struct vm_object *src,
+	off_t src_offset,
+	size_t count,
+	size_t *nr_copied);
+extern kern_status_t vm_object_transfer(
+	struct vm_object *dst,
+	off_t dst_offset,
+	struct vm_object *src,
+	off_t src_offset,
+	size_t count,
+	size_t *nr_moved);
+
+DEFINE_OBJECT_LOCK_FUNCTION(vm_object, vo_base)
+
+#endif

include/kernel/vm.h

@@ -1,14 +1,14 @@
-#ifndef MANGO_VM_H_
+#ifndef KERNEL_VM_H_
-#define MANGO_VM_H_
+#define KERNEL_VM_H_
 
-#include <stddef.h>
+#include <kernel/bitmap.h>
-#include <mango/types.h>
+#include <kernel/btree.h>
+#include <kernel/locks.h>
+#include <kernel/machine/vm.h>
+#include <kernel/queue.h>
+#include <kernel/types.h>
 #include <mango/status.h>
-#include <mango/queue.h>
+#include <stddef.h>
-#include <mango/btree.h>
-#include <mango/bitmap.h>
-#include <mango/locks.h>
-#include <mango/machine/vm.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -17,13 +17,13 @@ extern "C" {
 struct bcache;
 
 /* maximum number of NUMA nodes */
-#define VM_MAX_NODES 64
+#define VM_MAX_NODES            64
 /* maximum number of memory zones per node */
-#define VM_MAX_ZONES (VM_ZONE_MAX + 1)
+#define VM_MAX_ZONES            (VM_ZONE_MAX + 1)
 /* maximum number of supported page orders */
-#define VM_MAX_PAGE_ORDERS (VM_PAGE_MAX_ORDER + 1)
+#define VM_MAX_PAGE_ORDERS      (VM_PAGE_MAX_ORDER + 1)
 /* maximum number of sparse memory sectors */
-#define VM_MAX_SECTORS 8192
+#define VM_MAX_SECTORS          8192
 
 /* maximum number of disk sectors that can be stored in a single
    page. AKA the number of bits in the sector bitmap.
@@ -33,44 +33,32 @@ struct bcache;
 #define VM_CHECK_ALIGN(p, mask) ((((p) & (mask)) == (p)) ? 1 : 0)
 
 #define VM_CACHE_INITIALISED(c) ((c)->c_obj_count != 0)
-#define VM_PAGE_IS_FREE(pg) (((pg)->p_flags & (VM_PAGE_RESERVED | VM_PAGE_ALLOC)) == 0)
+#define VM_PAGE_IS_FREE(pg)                                                    \
+	(((pg)->p_flags & (VM_PAGE_RESERVED | VM_PAGE_ALLOC)) == 0)
 
-#define vm_page_foreach(pg, i) \
+#define vm_page_foreach(pg, i)                                                 \
 	for (struct vm_page *i = (pg); i; i = vm_page_get_next_tail(i))
 
 typedef phys_addr_t vm_alignment_t;
 typedef unsigned int vm_node_id_t;
 
-struct vm_object {
-	unsigned int reserved;
-};
-
 enum vm_model {
 	VM_MODEL_FLAT = 1,
 	VM_MODEL_SPARSE,
 };
 
-enum vm_prot {
-	VM_PROT_READ    = 0x01u,
-	VM_PROT_WRITE   = 0x02u,
-	VM_PROT_EXEC    = 0x04u,
-	VM_PROT_USER    = 0x08u,
-	VM_PROT_SVR     = 0x10u,
-	VM_PROT_NOCACHE = 0x20u,
-};
-
 enum vm_flags {
-	VM_NORMAL  = 0x00u,
+	VM_NORMAL = 0x00u,
 	VM_GET_DMA = 0x01u,
 };
 
 enum vm_zone_id {
-	/* NOTE that these are used as indices into the node_zones array in vm/zone.c
+	/* NOTE that these are used as indices into the node_zones array in
-	   they need to be continuous, and must start at 0!
+	   vm/zone.c they need to be continuous, and must start at 0!
 
 	   not all of these zones are implemented for every architecture. */
-	VM_ZONE_DMA     = 0u,
+	VM_ZONE_DMA = 0u,
-	VM_ZONE_NORMAL  = 1u,
+	VM_ZONE_NORMAL = 1u,
 	VM_ZONE_HIGHMEM = 2u,
 };
 
@@ -108,27 +96,28 @@ enum vm_page_order {
 };
 
 enum vm_page_flags {
-	/* page is reserved (probably by a call to memblock_reserve()) and cannot be
+	/* page is reserved (probably by a call to memblock_reserve()) and
-	   returned by any allocation function */
+	   cannot be returned by any allocation function */
-	VM_PAGE_RESERVED   = 0x01u,
+	VM_PAGE_RESERVED = 0x01u,
 	/* page has been allocated by a zone's buddy allocator, and is in-use */
-	VM_PAGE_ALLOC      = 0x02u,
+	VM_PAGE_ALLOC = 0x02u,
 	/* page is the first page of a huge-page */
-	VM_PAGE_HEAD       = 0x04u,
+	VM_PAGE_HEAD = 0x04u,
 	/* page is part of a huge-page */
-	VM_PAGE_HUGE       = 0x08u,
+	VM_PAGE_HUGE = 0x08u,
-	/* page is holding cached data from secondary storage, and can be freed if necessary (and not dirty). */
+	/* page is holding cached data from secondary storage, and can be freed
-	VM_PAGE_CACHE      = 0x10u,
+	 * if necessary (and not dirty). */
+	VM_PAGE_CACHE = 0x10u,
 };
 
 enum vm_memory_region_status {
-	VM_REGION_FREE		= 0x01u,
+	VM_REGION_FREE = 0x01u,
-	VM_REGION_RESERVED	= 0x02u,
+	VM_REGION_RESERVED = 0x02u,
 };
 
 enum vm_cache_flags {
 	VM_CACHE_OFFSLAB = 0x01u,
-	VM_CACHE_DMA     = 0x02u
+	VM_CACHE_DMA = 0x02u
 };
 
 struct vm_zone_descriptor {
@@ -151,12 +140,6 @@ struct vm_pg_data {
 	struct vm_zone pg_zones[VM_MAX_ZONES];
 };
 
-struct vm_region {
-	enum vm_memory_region_status r_status;
-	phys_addr_t r_base;
-	phys_addr_t r_limit;
-};
-
 struct vm_cache {
 	const char *c_name;
 	enum vm_cache_flags c_flags;
@@ -204,7 +187,7 @@ struct vm_slab {
 	     - s_freelist[s_free] should be set to the previous value of s_free.
 	   this is commented as it as flexible arrays are not supported in c++.
 	   */
-	//unsigned int s_freelist[];
+	// unsigned int s_freelist[];
 };
 
 struct vm_page {
@@ -231,20 +214,25 @@ struct vm_page {
 	/* multi-purpose list/tree entry.
 	   the owner of the page can decide what to do with this.
 	   some examples:
-	     - the buddy allocator uses this to maintain its per-zone free-page lists.
+	     - the buddy allocator uses this to maintain its per-zone free-page
-	     - the block cache uses this to maintain a tree of pages keyed by block number.
+	       lists.
-        */
+	     - vm_object uses it to maintain a btree of allocated pages keyed
+	       by offset/size.
+	     - the block cache uses this to maintain a tree of pages keyed by
+	       block number.
+	*/
 	union {
 		struct queue_entry p_list;
 		struct btree_node p_bnode;
 
-		/* btree_node contains three pointers, so provide three pointer-sized integers for
+		/* btree_node contains three pointers, so provide three
-		   use if p_bnode isn't needed. */
+		   pointer-sized integers for use if p_bnode isn't needed. */
 		uintptr_t priv1[3];
 	};
 
 	union {
-		/* used by bcache when sector size is < page size. bitmap of present/missing sectors */
+		/* used by bcache when sector size is < page size. bitmap of
+		 * present/missing sectors */
 		DECLARE_BITMAP(p_blockbits, VM_MAX_SECTORS_PER_PAGE);
 		uint32_t p_priv2;
 	};
@@ -252,10 +240,12 @@ struct vm_page {
 	union {
 		/* sector address, used by bcache */
 		sectors_t p_blockid;
+		/* offset of this page within the vm_object it is a part of */
+		off_t p_vmo_offset;
 
 		uint32_t p_priv3[2];
 	};
-} __attribute__((aligned(2 * sizeof(unsigned long))));
+} __aligned(2 * sizeof(unsigned long));
 
 /* represents a sector of memory, containing its own array of vm_pages.
    this struct is used under the sparse memory model, instead of the
@@ -272,39 +262,58 @@ struct vm_sector {
 	struct vm_page *s_pages;
 };
 
-extern kern_status_t vm_bootstrap(const struct vm_zone_descriptor *zones, size_t nr_zones);
+extern kern_status_t vm_bootstrap(
+	const struct vm_zone_descriptor *zones,
+	size_t nr_zones);
 extern enum vm_model vm_memory_model(void);
 extern void vm_set_memory_model(enum vm_model model);
 
 extern struct vm_pg_data *vm_pg_data_get(vm_node_id_t node);
 
-extern phys_addr_t vm_virt_to_phys(void *p);
+extern phys_addr_t vm_virt_to_phys(const void *p);
 extern void *vm_phys_to_virt(phys_addr_t p);
 
-extern void vm_page_init_array();
+extern size_t vm_page_order_to_bytes(enum vm_page_order order);
+extern size_t vm_page_order_to_pages(enum vm_page_order order);
+extern vm_alignment_t vm_page_order_to_alignment(enum vm_page_order order);
+extern void vm_page_init_array(void);
 extern struct vm_page *vm_page_get(phys_addr_t addr);
 extern phys_addr_t vm_page_get_paddr(struct vm_page *pg);
 extern struct vm_zone *vm_page_get_zone(struct vm_page *pg);
 extern void *vm_page_get_vaddr(struct vm_page *pg);
 extern size_t vm_page_get_pfn(struct vm_page *pg);
-extern size_t vm_page_order_to_bytes(enum vm_page_order order);
+static inline size_t vm_page_get_size_bytes(const struct vm_page *pg)
-extern size_t vm_page_order_to_pages(enum vm_page_order order);
+{
-extern vm_alignment_t vm_page_order_to_alignment(enum vm_page_order order);
+	return vm_page_order_to_bytes(pg->p_order);
-extern struct vm_page *vm_page_alloc(enum vm_page_order order, enum vm_flags flags);
+}
+extern struct vm_page *vm_page_alloc(
+	enum vm_page_order order,
+	enum vm_flags flags);
 extern void vm_page_free(struct vm_page *pg);
 
-extern int vm_page_split(struct vm_page *pg, struct vm_page **a, struct vm_page **b);
+extern int vm_page_split(
+	struct vm_page *pg,
+	struct vm_page **a,
+	struct vm_page **b);
 extern struct vm_page *vm_page_merge(struct vm_page *a, struct vm_page *b);
 extern struct vm_page *vm_page_get_buddy(struct vm_page *pg);
 extern struct vm_page *vm_page_get_next_tail(struct vm_page *pg);
 
 extern size_t vm_bytes_to_pages(size_t bytes);
 
-extern void vm_zone_init(struct vm_zone *z, const struct vm_zone_descriptor *zone_info);
+extern void vm_zone_init(
-extern struct vm_page *vm_zone_alloc_page(struct vm_zone *z, enum vm_page_order order, enum vm_flags flags);
+	struct vm_zone *z,
+	const struct vm_zone_descriptor *zone_info);
+extern struct vm_page *vm_zone_alloc_page(
+	struct vm_zone *z,
+	enum vm_page_order order,
+	enum vm_flags flags);
 extern void vm_zone_free_page(struct vm_zone *z, struct vm_page *pg);
 
-extern struct vm_cache *vm_cache_create(const char *name, size_t objsz, enum vm_cache_flags flags);
+extern struct vm_cache *vm_cache_create(
+	const char *name,
+	size_t objsz,
+	enum vm_cache_flags flags);
 extern void vm_cache_init(struct vm_cache *cache);
 extern void vm_cache_destroy(struct vm_cache *cache);
 extern void *vm_cache_alloc(struct vm_cache *cache, enum vm_flags flags);
@@ -330,15 +339,18 @@ extern size_t vm_page_get_pfn_sparse(struct vm_page *pg);
 #endif
 
 #ifdef __cplusplus
-inline void *operator new(size_t count, void *p) { return p; }
+inline void *operator new(size_t count, void *p)
+{
+	return p;
+}
 
-#define kmalloc_object(objtype, flags, ...)        \
+#define kmalloc_object(objtype, flags, ...)                                    \
-        __extension__({                            \
+	__extension__({                                                        \
-	void *p = kmalloc(sizeof(objtype), flags); \
+		void *p = kmalloc(sizeof(objtype), flags);                     \
-	if (p) {                                   \
+		if (p) {                                                       \
-		new (p) objtype(__VA_ARGS__);      \
+			new (p) objtype(__VA_ARGS__);                          \
-	}                                          \
+		}                                                              \
-	(objtype *)p;                              \
+		(objtype *)p;                                                  \
 	})
 
 #endif

include/kernel/wait.h

@@ -0,0 +1,45 @@
+#ifndef KERNEL_WAIT_H_
+#define KERNEL_WAIT_H_
+
+#include <kernel/locks.h>
+#include <kernel/queue.h>
+
+#define wait_event(wq, cond)                                                   \
+	({                                                                     \
+		struct thread *self = current_thread();                        \
+		struct wait_item waiter;                                       \
+		wait_item_init(&waiter, self);                                 \
+		for (;;) {                                                     \
+			thread_wait_begin(&waiter, wq);                        \
+			if (cond) {                                            \
+				break;                                         \
+			}                                                      \
+			schedule(SCHED_NORMAL);                                \
+		}                                                              \
+		thread_wait_end(&waiter, wq);                                  \
+	})
+
+struct wait_item {
+	struct thread *w_thread;
+	struct queue_entry w_entry;
+};
+
+struct waitqueue {
+	struct queue wq_waiters;
+	spin_lock_t wq_lock;
+};
+
+extern void wait_item_init(struct wait_item *item, struct thread *thr);
+extern void thread_wait_begin(struct wait_item *waiter, struct waitqueue *q);
+extern void thread_wait_end(struct wait_item *waiter, struct waitqueue *q);
+extern void thread_wait_begin_nosleep(
+	struct wait_item *waiter,
+	struct waitqueue *q);
+extern void thread_wait_end_nosleep(
+	struct wait_item *waiter,
+	struct waitqueue *q);
+extern void wait_on_queue(struct waitqueue *q);
+extern void wakeup_queue(struct waitqueue *q);
+extern void wakeup_one(struct waitqueue *q);
+
+#endif

include/kernel/work.h

@@ -0,0 +1,37 @@
+#ifndef KERNEL_WORK_H_
+#define KERNEL_WORK_H_
+
+#include <kernel/locks.h>
+#include <kernel/queue.h>
+#include <stddef.h>
+
+struct work_item;
+
+typedef void (*work_func_t)(struct work_item *);
+
+struct work_item {
+	void *w_data;
+	work_func_t w_func;
+	struct queue_entry w_head;
+};
+
+struct worker_pool {
+	struct thread **wp_workers;
+	size_t wp_nworkers;
+};
+
+struct workqueue {
+	spin_lock_t wq_lock;
+	struct queue wq_queue; /* list of struct work_item */
+};
+
+extern void work_item_init(work_func_t func, void *data, struct work_item *out);
+extern void workqueue_init(struct workqueue *wq);
+extern struct worker_pool *worker_pool_create(size_t nworkers);
+extern struct worker_pool *global_worker_pool(void);
+extern bool schedule_work_on(struct workqueue *wq, struct work_item *work);
+extern bool schedule_work(struct work_item *work);
+
+extern void wake_workers(struct workqueue *wq, struct worker_pool *pool);
+
+#endif

include/mango/locks.h

@@ -1,25 +0,0 @@
-#ifndef MANGO_LOCKS_H_
-#define MANGO_LOCKS_H_
-
-#include <mango/compiler.h>
-#include <mango/machine/hwlock.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef __aligned(8) ml_hwlock_t spin_lock_t;
-
-#define SPIN_LOCK_INIT ML_HWLOCK_INIT
-
-#define spin_lock(lck) ml_hwlock_lock(lck);
-#define spin_unlock(lck) ml_hwlock_unlock(lck);
-
-#define spin_lock_irqsave(lck, flags) ml_hwlock_lock_irqsave(lck, flags);
-#define spin_unlock_irqrestore(lck, flags) ml_hwlock_unlock_irqrestore(lck, flags);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

include/mango/object.h

@@ -1,71 +0,0 @@
-#ifndef MANGO_OBJECT_H_
-#define MANGO_OBJECT_H_
-
-#include <mango/flags.h>
-#include <mango/locks.h>
-#include <mango/status.h>
-#include <mango/vm.h>
-#include <stddef.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define OBJECT_MAGIC    0xBADDCAFE
-#define OBJECT_NAME_MAX 64
-#define OBJECT_PATH_MAX 256
-
-#define OBJECT_CAST(to_type, to_type_member, p)                                \
-	((to_type *)((uintptr_t)p) - offsetof(to_type, to_type_member))
-#define OBJECT_C_CAST(c_type, c_type_member, obj_type, objp)                   \
-	OBJECT_IS_TYPE(objp, obj_type)                                         \
-	? OBJECT_CAST(c_type, c_type_member, (objp)) : NULL
-#define OBJECT_IS_TYPE(obj, type_ptr) ((obj)->ob_type == (type_ptr))
-
-struct object;
-struct object_attrib;
-
-enum object_type_flags {
-	OBJTYPE_INIT = 0x01u,
-};
-
-struct object_ops {
-	kern_status_t (*destroy)(struct object *obj);
-};
-
-struct object_type {
-	enum object_type_flags ob_flags;
-	char ob_name[32];
-	unsigned int ob_size;
-	unsigned int ob_header_offset;
-	struct vm_cache ob_cache;
-	struct queue_entry ob_list;
-	struct object_ops ob_ops;
-};
-
-struct object {
-	uint32_t ob_magic;
-	struct object_type *ob_type;
-	spin_lock_t ob_lock;
-	unsigned int ob_refcount;
-	unsigned int ob_handles;
-	struct queue_entry ob_list;
-} __aligned(sizeof(long));
-
-extern kern_status_t object_bootstrap(void);
-extern kern_status_t object_type_register(struct object_type *p);
-extern kern_status_t object_type_unregister(struct object_type *p);
-
-extern struct object *object_create(struct object_type *type);
-extern struct object *object_ref(struct object *obj);
-extern void object_deref(struct object *obj);
-extern void object_lock(struct object *obj);
-extern void object_unlock(struct object *obj);
-extern void object_lock_irqsave(struct object *obj, unsigned long *flags);
-extern void object_unlock_irqrestore(struct object *obj, unsigned long flags);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

include/mango/panic.h

@@ -1,12 +0,0 @@
-#ifndef MANGO_PANIC_H_
-#define MANGO_PANIC_H_
-
-#include <mango/compiler.h>
-
-struct cpu_context;
-
-#define panic(...) panic_irq(NULL, __VA_ARGS__)
-
-extern void __noreturn panic_irq(struct cpu_context *ctx, const char *fmt, ...);
-
-#endif

include/mango/pmap.h

@@ -1,42 +0,0 @@
-#ifndef MANGO_PMAP_H_
-#define MANGO_PMAP_H_
-
-/* all the functions declared in this file are defined in arch/xyz/pmap.c */
-
-#include <mango/vm.h>
-#include <mango/status.h>
-#include <mango/machine/pmap.h>
-#include <stddef.h>
-
-#define PFN(x) ((x) >> VM_PAGE_SHIFT)
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef ml_pmap_t pmap_t;
-typedef ml_pfn_t pfn_t;
-
-enum pmap_flags {
-	PMAP_NORMAL = 0x00u,
-	PMAP_HUGEPAGE = 0x01u,
-};
-
-extern void pmap_bootstrap(void);
-extern pmap_t get_kernel_pmap(void);
-
-extern pmap_t pmap_create(void);
-extern void pmap_destroy(pmap_t pmap);
-extern void pmap_switch(pmap_t pmap);
-
-extern kern_status_t pmap_add(pmap_t pmap, void *p, pfn_t pfn, enum vm_prot prot, enum pmap_flags flags);
-extern kern_status_t pmap_add_block(pmap_t pmap, void *p, pfn_t pfn, size_t len, enum vm_prot prot, enum pmap_flags flags);
-
-extern kern_status_t pmap_remove(pmap_t pmap, void *p);
-extern kern_status_t pmap_remove_range(pmap_t pmap, void *p, size_t len);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

include/mango/printk.h

@@ -1,17 +0,0 @@
-#ifndef MANGO_PRINTK_H_
-#define MANGO_PRINTK_H_
-
-#include <mango/console.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-extern void early_printk_init(struct console *con);
-extern int printk(const char *format, ...);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

include/mango/sched.h

@@ -1,246 +0,0 @@
-#ifndef MANGO_SCHED_H_
-#define MANGO_SCHED_H_
-
-#include <mango/btree.h>
-#include <mango/locks.h>
-#include <mango/object.h>
-#include <mango/pmap.h>
-#include <mango/queue.h>
-#include <mango/status.h>
-
-#define TASK_NAME_MAX       64
-#define PRIO_MAX            32
-#define THREAD_KSTACK_ORDER VM_PAGE_4K
-#define THREAD_MAX          65536
-
-#define wait_event(wq, cond)                                                   \
-	({                                                                     \
-		struct thread *self = current_thread();                        \
-		struct wait_item waiter;                                       \
-		wait_item_init(&waiter, self);                                 \
-		for (;;) {                                                     \
-			thread_wait_begin(&waiter, wq);                        \
-			if (cond) {                                            \
-				break;                                         \
-			}                                                      \
-			schedule(SCHED_NORMAL);                                \
-		}                                                              \
-		thread_wait_end(&waiter, wq);                                  \
-	})
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-struct runqueue;
-struct work_item;
-
-enum task_state {
-	TASK_RUNNING,
-	TASK_STOPPED,
-};
-
-enum thread_state {
-	THREAD_READY = 1,
-	THREAD_SLEEPING = 2,
-	THREAD_STOPPED = 3,
-};
-
-enum thread_flags {
-	THREAD_F_NEED_RESCHED = 0x01u,
-	THREAD_F_NO_PREEMPT = 0x02u,
-};
-
-enum sched_priority {
-	PRIO_IDLE = 4,
-	PRIO_SUBNORMAL = 6,
-	PRIO_NORMAL = 10,
-	PRIO_SUPERNORMAL = 14,
-	PRIO_HIGH = 18,
-	PRIO_REALTIME = 24,
-};
-
-enum sched_mode {
-	/* used when calling from non-interrupt context.
-	   threads that aren't in state THREAD_READY are
-	   removed from the runqueue. */
-	SCHED_NORMAL = 0,
-	/* used when calling from interrupt context.
-	   threads that aren't in state THREAD_READY are
-	   still added to the runqueue. */
-	SCHED_IRQ = 1,
-};
-
-struct task {
-	struct object t_base;
-
-	struct task *t_parent;
-	unsigned int t_id;
-	enum task_state t_state;
-	char t_name[TASK_NAME_MAX];
-
-	pmap_t t_pmap;
-
-	struct btree_node t_tasklist;
-	struct queue t_threads;
-	struct queue t_children;
-};
-
-struct thread {
-	struct object thr_base;
-
-	enum thread_state tr_state;
-	enum thread_flags tr_flags;
-	struct task *tr_parent;
-
-	unsigned int tr_id;
-	unsigned int tr_prio;
-
-	cycles_t tr_charge_period_start;
-	cycles_t tr_quantum_cycles, tr_quantum_target;
-	cycles_t tr_total_cycles;
-
-	uintptr_t tr_sp, tr_bp;
-
-	struct runqueue *tr_rq;
-
-	struct queue_entry tr_threads;
-	struct queue_entry tr_rqentry;
-
-	struct vm_page *tr_kstack;
-};
-
-struct runqueue {
-	struct queue rq_queues[PRIO_MAX];
-	uint32_t rq_readybits;
-	spin_lock_t rq_lock;
-	unsigned int rq_nthreads;
-
-	struct thread *rq_cur, *rq_idle;
-};
-
-struct timer {
-	struct queue_entry t_entry;
-	struct cpu_data *t_cpu;
-	struct thread *t_owner;
-	unsigned long t_expiry;
-	void (*t_callback)(struct timer *);
-};
-
-struct wait_item {
-	struct thread *w_thread;
-	struct queue_entry w_entry;
-};
-
-struct waitqueue {
-	struct queue wq_waiters;
-	spin_lock_t wq_lock;
-};
-
-typedef void (*work_func_t)(struct work_item *);
-
-struct work_item {
-	void *w_data;
-	work_func_t w_func;
-	struct queue_entry w_head;
-};
-
-struct worker_pool {
-	struct thread **wp_workers;
-	size_t wp_nworkers;
-};
-
-struct workqueue {
-	spin_lock_t wq_lock;
-	struct queue wq_queue; /* list of struct work_item */
-};
-
-extern kern_status_t sched_init(void);
-extern void schedule(enum sched_mode mode);
-extern void preempt_disable(void);
-extern void preempt_enable(void);
-
-extern void rq_init(struct runqueue *rq);
-extern struct thread *rq_dequeue(struct runqueue *rq);
-extern void rq_enqueue(struct runqueue *rq, struct thread *thr);
-static inline void rq_lock(struct runqueue *rq, unsigned long *flags)
-{
-	spin_lock_irqsave(&rq->rq_lock, flags);
-}
-static inline void rq_unlock(struct runqueue *rq, unsigned long flags)
-{
-	spin_unlock_irqrestore(&rq->rq_lock, flags);
-}
-extern void rq_remove_thread(struct runqueue *rq, struct thread *thr);
-extern struct runqueue *cpu_rq(unsigned int cpu);
-
-extern struct task *task_alloc(void);
-static inline struct task *task_ref(struct task *task)
-{
-	return OBJECT_CAST(struct task, t_base, object_ref(&task->t_base));
-}
-static inline void task_deref(struct task *task)
-{
-	object_deref(&task->t_base);
-}
-extern struct task *task_from_pid(unsigned int pid);
-extern struct task *kernel_task(void);
-extern struct task *idle_task(void);
-extern cycles_t default_quantum(void);
-
-extern bool need_resched(void);
-extern struct task *current_task(void);
-extern struct thread *current_thread(void);
-
-extern struct runqueue *select_rq_for_thread(struct thread *thr);
-extern void schedule_thread_on_cpu(struct thread *thr);
-
-extern void start_charge_period(void);
-extern void end_charge_period(void);
-
-static inline void task_lock_irqsave(struct task *task, unsigned long *flags)
-{
-	object_lock_irqsave(&task->t_base, flags);
-}
-
-static inline void task_unlock_irqrestore(
-	struct task *task,
-	unsigned long flags)
-{
-	object_unlock_irqrestore(&task->t_base, flags);
-}
-
-extern struct thread *thread_alloc(void);
-extern kern_status_t thread_init(struct thread *thr, uintptr_t ip);
-extern int thread_priority(struct thread *thr);
-extern void idle(void);
-extern struct thread *create_kernel_thread(void (*fn)(void));
-extern struct thread *create_idle_thread(void);
-
-extern void add_timer(struct timer *timer);
-extern void remove_timer(struct timer *timer);
-extern unsigned long schedule_timeout(unsigned long clock_ticks);
-extern unsigned long milli_sleep(unsigned long ms);
-extern void sleep_forever(void);
-
-extern void wait_item_init(struct wait_item *item, struct thread *thr);
-extern void thread_wait_begin(struct wait_item *waiter, struct waitqueue *q);
-extern void thread_wait_end(struct wait_item *waiter, struct waitqueue *q);
-extern void wait_on_queue(struct waitqueue *q);
-extern void wakeup_queue(struct waitqueue *q);
-extern void wakeup_one(struct waitqueue *q);
-
-extern void work_item_init(work_func_t func, void *data, struct work_item *out);
-extern void workqueue_init(struct workqueue *wq);
-extern struct worker_pool *worker_pool_create(size_t nworkers);
-extern struct worker_pool *global_worker_pool(void);
-extern bool schedule_work_on(struct workqueue *wq, struct work_item *work);
-extern bool schedule_work(struct work_item *work);
-
-extern void wake_workers(struct workqueue *wq, struct worker_pool *pool);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif