kernel: implementing mapping and execution of bsp executable

this allows user-accessible page mappings to be created. for kernel memory
mappings, PTE_USR will only be cleared on the lowest-level table entry.

arch/user/include/mango/machine/init.h

@@ -1,6 +1,7 @@
 #ifndef MANGO_X86_64_INIT_H_
 #define MANGO_X86_64_INIT_H_
 
+#include <stddef.h>
 #include <stdint.h>
 
 #ifdef __cplusplus
@@ -8,16 +9,18 @@ extern "C" {
 #endif
 
 #define __X2(x) #x
-#define __X(x) __X2(x)
+#define __X(x)  __X2(x)
 
 #ifdef __APPLE__
-#define __define_initcall(fn, id) \
-	static initcall_t __initcall_##fn##id __used \
-	__section("__DATA,__initcall" __X(id) ".init") = (fn)
+#define __define_initcall(fn, id)                                              \
+	static initcall_t __initcall_##fn##id __used __section(                \
+		"__DATA,__initcall" __X(id) ".init")                           \
+		= (fn)
 #else
-#define __define_initcall(fn, id) \
-	static initcall_t __initcall_##fn##id __used \
-	__section("initcall" __X(id) "_init") = (fn)
+#define __define_initcall(fn, id)                                              \
+	static initcall_t __initcall_##fn##id __used __section(                \
+		"initcall" __X(id) "_init")                                    \
+		= (fn)
 #endif
 
 extern int ml_init(uintptr_t arg);

arch/user/include/mango/machine/pmap.h

@@ -1,6 +1,8 @@
 #ifndef MANGO_USER_PMAP_H_
 #define MANGO_USER_PMAP_H_
 
+#include <stdint.h>
+
 typedef uintptr_t ml_pmap_t;
 typedef uint64_t ml_pfn_t;
 

arch/x86_64/cpu.c

@@ -1,11 +1,14 @@
-#include <mango/machine/cpu.h>
 #include <arch/msr.h>
+#include <mango/machine/cpu.h>
 
 int ml_cpu_block_init(ml_cpu_block *p)
 {
 	p->c_this = p;
 	gdt_init(&p->c_gdt, &p->c_gdt_ptr);
 	idt_init(&p->c_idt_ptr);
+	tss_init(&p->c_tss, &p->c_tss_ptr);
+
+	gdt_write_tss(&p->c_gdt, &p->c_tss);
 	return 0;
 }
 
@@ -13,6 +16,27 @@ int ml_cpu_block_use(ml_cpu_block *p)
 {
 	gdt_load(&p->c_gdt_ptr);
 	idt_load(&p->c_idt_ptr);
+	tss_load(&p->c_tss);
 	wrmsr(MSR_GS_BASE, (uint64_t)p);
 	return 0;
 }
+
+virt_addr_t ml_cpu_block_get_kstack(ml_cpu_block *p)
+{
+	return tss_get_kstack(&p->c_tss);
+}
+
+virt_addr_t ml_cpu_block_get_ustack(ml_cpu_block *p)
+{
+	return tss_get_ustack(&p->c_tss);
+}
+
+void ml_cpu_block_set_kstack(ml_cpu_block *p, virt_addr_t sp)
+{
+	tss_set_kstack(&p->c_tss, sp);
+}
+
+void ml_cpu_block_set_ustack(ml_cpu_block *p, virt_addr_t sp)
+{
+	tss_set_ustack(&p->c_tss, sp);
+}

arch/x86_64/gdt.c

@@ -1,5 +1,8 @@
-#include <mango/libc/string.h>
 #include <arch/gdt.h>
+#include <arch/tss.h>
+#include <mango/libc/string.h>
+#include <mango/types.h>
+#include <stddef.h>
 
 static void init_entry(struct gdt_entry *entry, int access, int flags)
 {
@@ -15,13 +18,42 @@ int gdt_init(struct gdt *gdt, struct gdt_ptr *gdtp)
 {
 	memset(&gdt->g_entries[0], 0x0, sizeof gdt->g_entries[0]);
 
-	init_entry(&gdt->g_entries[1], GDT_A_PRESENT | GDT_A_CODEREAD | GDT_A_CODE, GDT_F_64BIT);
-	init_entry(&gdt->g_entries[2], GDT_A_PRESENT | GDT_A_DATAWRITE | GDT_A_DATA, GDT_F_64BIT);
-	init_entry(&gdt->g_entries[3], GDT_A_PRESENT | GDT_A_USER | GDT_A_CODEREAD | GDT_A_CODE, GDT_F_64BIT);
-	init_entry(&gdt->g_entries[4], GDT_A_PRESENT | GDT_A_USER | GDT_A_DATAWRITE | GDT_A_DATA, GDT_F_64BIT);
+	init_entry(
+		&gdt->g_entries[1],
+		GDT_A_PRESENT | GDT_A_CODEREAD | GDT_A_CODE,
+		GDT_F_64BIT);
+	init_entry(
+		&gdt->g_entries[2],
+		GDT_A_PRESENT | GDT_A_DATAWRITE | GDT_A_DATA,
+		GDT_F_64BIT);
+	init_entry(
+		&gdt->g_entries[3],
+		GDT_A_PRESENT | GDT_A_USER | GDT_A_CODEREAD | GDT_A_CODE,
+		GDT_F_64BIT);
+	init_entry(
+		&gdt->g_entries[4],
+		GDT_A_PRESENT | GDT_A_USER | GDT_A_DATAWRITE | GDT_A_DATA,
+		GDT_F_64BIT);
 
 	gdtp->g_ptr = (uint64_t)gdt;
 	gdtp->g_limit = sizeof(*gdt) - 1;
 
 	return 0;
 }
+
+void gdt_write_tss(struct gdt *gdt, struct tss *tss)
+{
+	struct tss_gdt_entry *tss_entry = &gdt->g_tss;
+
+	virt_addr_t base = (virt_addr_t)tss;
+	size_t limit = sizeof *tss;
+
+	tss_entry->ge_base_low = (base & 0xFFFF);
+	tss_entry->ge_base_mid = (base >> 16) & 0xFF;
+	tss_entry->ge_base_hi = (base >> 24) & 0xFF;
+	tss_entry->ge_base_ext = (base >> 32) & 0xFFFFFFFF;
+	tss_entry->ge_limit_low = (limit & 0xFFFF);
+	tss_entry->ge_gran = (limit >> 16) & 0xF;
+	tss_entry->ge_access = 0xE9;
+	tss_entry->ge_reserved = 0;
+}

arch/x86_64/include/arch/gdt.h

@@ -1,6 +1,7 @@
 #ifndef ARCH_GDT_H_
 #define ARCH_GDT_H_
 
+#include <arch/tss.h>
 #include <mango/compiler.h>
 #include <stdint.h>
 
@@ -8,19 +9,19 @@
 extern "C" {
 #endif
 
-#define NR_GDT_ENTRIES 5
+#define NR_GDT_ENTRIES  5
 
-#define GDT_A_PRESENT		(1 << 7)
-#define GDT_A_USER		(3 << 5)
-#define GDT_A_CODE		(3 << 3)
-#define GDT_A_DATA		(8 << 1)
-#define GDT_A_RPLK		(1 << 5)
-#define GDT_A_CODEREAD		(1 << 1)
-#define GDT_A_DATAWRITE		(1 << 1)
-#define GDT_A_GROWUP		(1 << 5)
-#define GDT_F_64BIT		(0x2F)
-#define GDT_F_32BIT		(0x4F)
-#define GDT_F_16BIT		(0xF)
+#define GDT_A_PRESENT   (1 << 7)
+#define GDT_A_USER      (3 << 5)
+#define GDT_A_CODE      (3 << 3)
+#define GDT_A_DATA      (8 << 1)
+#define GDT_A_RPLK      (1 << 5)
+#define GDT_A_CODEREAD  (1 << 1)
+#define GDT_A_DATAWRITE (1 << 1)
+#define GDT_A_GROWUP    (1 << 5)
+#define GDT_F_64BIT     (0x2F)
+#define GDT_F_32BIT     (0x4F)
+#define GDT_F_16BIT     (0xF)
 
 struct gdt_entry {
 	uint16_t ge_limit_low;
@@ -33,6 +34,7 @@ struct gdt_entry {
 
 struct gdt {
 	struct gdt_entry g_entries[NR_GDT_ENTRIES];
+	struct tss_gdt_entry g_tss;
 } __packed;
 
 struct gdt_ptr {
@@ -41,6 +43,7 @@ struct gdt_ptr {
 } __packed;
 
 extern int gdt_init(struct gdt *gdt, struct gdt_ptr *gdtp);
+extern void gdt_write_tss(struct gdt *gdt, struct tss *tss);
 extern int gdt_load(struct gdt_ptr *gdtp);
 
 #ifdef __cplusplus

arch/x86_64/include/arch/irq.h

@@ -11,6 +11,8 @@ extern "C" {
 
 #define NR_IDT_ENTRIES 256
 
+struct ml_cpu_context;
+
 enum irq_vector {
 	IRQ0 = 32,
 	IRQ1,
@@ -35,13 +37,6 @@ struct irq_hook {
 	int (*irq_callback)(void);
 };
 
-struct cpu_context {
-	uint64_t r15, r14, r13, r12, r11, r10, r9, r8;
-	uint64_t rdi, rsi, rbp, unused_rsp, rbx, rdx, rcx, rax;
-	uint64_t int_no, err_no;
-	uint64_t rip, cs, rflags, rsp, ss;
-} __packed;
-
 struct idt_entry {
 	uint16_t base_low;
 	uint16_t selector;
@@ -64,7 +59,7 @@ struct idt_ptr {
 	uintptr_t i_base;
 } __packed;
 
-typedef void (*int_hook)(struct cpu_context *);
+typedef void (*int_hook)(struct ml_cpu_context *);
 
 extern int idt_init(struct idt_ptr *idtp);
 extern int idt_load(struct idt_ptr *idtp);

arch/x86_64/include/arch/tss.h

@@ -0,0 +1,55 @@
+#ifndef ARCH_TSS_H_
+#define ARCH_TSS_H_
+
+#include <mango/compiler.h>
+#include <mango/types.h>
+#include <stdint.h>
+
+#define TSS_GDT_INDEX 5
+
+struct tss {
+	uint32_t res0;
+	uint64_t rsp0;
+	uint64_t rsp1;
+	uint64_t rsp2;
+	uint64_t res1;
+	uint64_t ist1;
+	uint64_t ist2;
+	uint64_t ist3;
+	uint64_t ist4;
+	uint64_t ist5;
+	uint64_t ist6;
+	uint64_t ist7;
+	uint64_t res2;
+	uint16_t res3;
+	uint16_t iomap_offset;
+} __packed;
+
+struct tss_gdt_entry {
+	/* these fields are copied from struct gdt_entry */
+	uint16_t ge_limit_low;
+	uint16_t ge_base_low;
+	uint8_t ge_base_mid;
+	uint8_t ge_access;
+	uint8_t ge_gran;
+	uint8_t ge_base_hi;
+	/* these fields are specific to the TSS entry */
+	uint32_t ge_base_ext;
+	uint32_t ge_reserved;
+} __packed;
+
+struct tss_ptr {
+	uint16_t tss_limit;
+	uint64_t tss_base;
+} __packed;
+
+extern void tss_init(struct tss *tss, struct tss_ptr *ptr);
+extern void tss_load(struct tss *tss);
+
+extern virt_addr_t tss_get_kstack(struct tss *tss);
+extern virt_addr_t tss_get_ustack(struct tss *tss);
+
+extern void tss_set_kstack(struct tss *tss, virt_addr_t sp);
+extern void tss_set_ustack(struct tss *tss, virt_addr_t sp);
+
+#endif

arch/x86_64/include/mango/machine/cpu.h

@@ -3,16 +3,24 @@
 
 #include <arch/gdt.h>
 #include <arch/irq.h>
+#include <arch/tss.h>
+#include <mango/types.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#define ML_BIG_ENDIAN 0
+#define ML_BIG_ENDIAN            0
 
-#define ml_cpu_block_get_id(p) ((p)->c_cpu_id)
+#define ml_cpu_block_get_id(p)   ((p)->c_cpu_id)
 #define ml_cpu_block_get_data(p) ((p)->c_data)
 
+#if 0
+#define ml_read_sp(sp, bp)                                                     \
+	asm volatile("mov %%rsp, %0" : "=r"(sp));                              \
+	asm volatile("mov %%rbp, %0" : "=r"(bp));
+#endif
+
 struct cpu_data;
 
 typedef struct ml_cpu_block {
@@ -21,23 +29,39 @@ typedef struct ml_cpu_block {
 	struct gdt c_gdt;
 	struct gdt_ptr c_gdt_ptr;
 
+	struct tss c_tss;
+	struct tss_ptr c_tss_ptr;
+
 	struct idt_ptr c_idt_ptr;
 	unsigned int c_cpu_id;
 
 	struct cpu_data *c_data;
 } ml_cpu_block;
 
-#define ml_cpu_pause() __asm__ __volatile__("hlt")
-#define ml_cpu_relax() __asm__ __volatile__("pause")
+struct ml_cpu_context {
+	uint64_t r15, r14, r13, r12, r11, r10, r9, r8;
+	uint64_t rdi, rsi, rbp, unused_rsp, rbx, rdx, rcx, rax;
+	uint64_t int_no, err_no;
+	uint64_t rip, cs, rflags, rsp, ss;
+} __packed;
+
+#define ml_cpu_pause()   __asm__ __volatile__("hlt")
+#define ml_cpu_relax()   __asm__ __volatile__("pause")
 
 #define ml_int_disable() __asm__ __volatile__("cli")
-#define ml_int_enable() __asm__ __volatile__("sti")
+#define ml_int_enable()  __asm__ __volatile__("sti")
 
 extern int ml_init_bootcpu(void);
 
 extern int ml_cpu_block_init(ml_cpu_block *p);
 extern int ml_cpu_block_use(ml_cpu_block *p);
 
+extern virt_addr_t ml_cpu_block_get_ustack(ml_cpu_block *p);
+extern virt_addr_t ml_cpu_block_get_kstack(ml_cpu_block *p);
+
+extern void ml_cpu_block_set_ustack(ml_cpu_block *p, virt_addr_t sp);
+extern void ml_cpu_block_set_kstack(ml_cpu_block *p, virt_addr_t sp);
+
 /* defined in cpu_ctrl.S */
 extern void ml_halt_cpu(void);
 extern ml_cpu_block *ml_this_cpu(void);

arch/x86_64/include/mango/machine/init.h

@@ -1,6 +1,7 @@
 #ifndef MANGO_X86_64_INIT_H_
 #define MANGO_X86_64_INIT_H_
 
+#include <stddef.h>
 #include <stdint.h>
 
 #ifdef __cplusplus
@@ -8,11 +9,12 @@ extern "C" {
 #endif
 
 #define __X2(x) #x
-#define __X(x) __X2(x)
+#define __X(x)  __X2(x)
 
-#define __define_initcall(fn, id) \
-	static initcall_t __initcall_##fn##id __used \
-	__section(".initcall" __X(id) ".init") = (fn)
+#define __define_initcall(fn, id)                                              \
+	static initcall_t __initcall_##fn##id __used __section(                \
+		".initcall" __X(id) ".init")                                   \
+		= (fn)
 
 extern int ml_init(uintptr_t arg);
 

arch/x86_64/include/mango/machine/panic.h

@@ -3,10 +3,10 @@
 
 #include <stdint.h>
 
-struct cpu_context;
+struct ml_cpu_context;
 
-extern void ml_print_cpu_state(struct cpu_context *ctx);
+extern void ml_print_cpu_state(struct ml_cpu_context *ctx);
 extern void ml_print_stack_trace(uintptr_t ip);
-extern void ml_print_stack_trace_irq(struct cpu_context *ctx);
+extern void ml_print_stack_trace_irq(struct ml_cpu_context *ctx);
 
 #endif

arch/x86_64/include/mango/machine/pmap.h

@@ -3,6 +3,8 @@
 
 #include <arch/paging.h>
 
+#define ML_PMAP_INVALID ((uintptr_t)-1)
+
 typedef pml4t_ptr_t ml_pmap_t;
 typedef uint64_t ml_pfn_t;
 

arch/x86_64/include/mango/machine/thread.h

@@ -3,8 +3,27 @@
 
 #include <mango/sched.h>
 
-extern void switch_to(struct thread *from, struct thread *to);
-extern void prepare_stack(uintptr_t ip, uintptr_t *sp);
-extern void user_jump(uintptr_t ip, uintptr_t sp);
+struct ml_cpu_context;
+
+/* switch from one thread to another. the stack of the `to` thread must have
+ * been prepared in one of two ways:
+ *  1) a previous call to ml_thread_switch where it was the `from` thread.
+ *  2) a call to ml_thread_prepare_kernel_context
+ * the switch occurs entirely with kernel mode. a further return from an
+ * interrupt context is then used to return to usermode if necessary.
+ */
+extern void ml_thread_switch(struct thread *from, struct thread *to);
+
+/* perform the initial transition to userspace. the stack must be prepared using
+ * ml_thread_prepare_user_context before this function can be used */
+extern void ml_thread_switch_user(void);
+/* prepare the stack so that ml_thread_switch can jump to the specified IP/SP */
+extern void ml_thread_prepare_kernel_context(uintptr_t ip, uintptr_t *sp);
+/* prepare the stack so that ml_thread_switch_user can jump to usermode
+ * with the specified IP/user SP */
+extern void ml_thread_prepare_user_context(
+	virt_addr_t ip,
+	virt_addr_t user_sp,
+	virt_addr_t *kernel_sp);
 
 #endif

arch/x86_64/include/mango/machine/vm.h

@@ -11,14 +11,20 @@
 #define VM_PAGEMAP_BASE   0xFFFF888000000000
 #define VM_PAGEMAP_LIMIT  0xFFFFC87FFFFFFFFF
 
-#define VM_PAGE_SIZE  0x1000
-#define VM_PAGE_MASK  (VM_PAGE_SIZE-1)
-#define VM_PAGE_SHIFT 12
+#define VM_PAGE_SIZE      0x1000
+#define VM_PAGE_MASK      (VM_PAGE_SIZE - 1)
+#define VM_PAGE_SHIFT     12
 
 #define VM_PAGE_MIN_ORDER VM_PAGE_4K
 #define VM_PAGE_MAX_ORDER VM_PAGE_8M
 
-#define VM_ZONE_MIN VM_ZONE_DMA
-#define VM_ZONE_MAX VM_ZONE_NORMAL
+#define VM_ZONE_MIN       VM_ZONE_DMA
+#define VM_ZONE_MAX       VM_ZONE_NORMAL
+
+#define VM_USER_BASE      0x0000000000100000
+#define VM_USER_LIMIT     0x00007fffffffffff
+
+#define VM_KERNEL_BASE    0XFFFF800000000000
+#define VM_KERNEL_LIMIT   0XFFFFFFFFFFFFFFFF
 
 #endif

arch/x86_64/init.c

@@ -3,6 +3,7 @@
 #include <arch/serial.h>
 #include <arch/vgacon.h>
 #include <mango/arg.h>
+#include <mango/bsp.h>
 #include <mango/clock.h>
 #include <mango/console.h>
 #include <mango/cpu.h>
@@ -30,7 +31,7 @@ static void bootstrap_cpu_init(void)
 	ml_cpu_block_use(&g_bootstrap_cpu);
 }
 
-static void early_vm_init(void)
+static void early_vm_init(uintptr_t reserve_end)
 {
 	uintptr_t alloc_start = VM_KERNEL_VOFFSET;
 	/* boot code mapped 2 GiB of memory from
@@ -42,10 +43,10 @@ static void early_vm_init(void)
 	       alloc_start,
 	       alloc_end);
 
-	memblock_reserve(0x00, (uintptr_t)__pend);
+	memblock_reserve(0x00, reserve_end);
 	printk("memblock: reserved bios+kernel at [0x%016llx-0x%016llx]",
 	       0,
-	       (uintptr_t)__pend);
+	       reserve_end);
 }
 
 void early_console_init(void)
@@ -70,6 +71,23 @@ static void use_uniprocessor_topology(void)
 	cpu_set_online(0);
 }
 
+static void find_bsp(multiboot_info_t *mb, struct boot_module *out)
+{
+	memset(out, 0x0, sizeof *out);
+
+	printk("modules=%u: %llx", mb->mods_count, mb->mods_addr);
+
+	multiboot_module_t *mods = PTR32(mb->mods_addr);
+	size_t nr_mods = mb->mods_count;
+
+	if (nr_mods < 1) {
+		return;
+	}
+
+	out->mod_base = mods[0].mod_start;
+	out->mod_size = mods[0].mod_end - mods[0].mod_start;
+}
+
 int ml_init(uintptr_t arg)
 {
 	multiboot_info_t *mb = (multiboot_info_t *)arg;
@@ -83,7 +101,16 @@ int ml_init(uintptr_t arg)
 
 	print_kernel_banner();
 
-	early_vm_init();
+	struct boot_module bsp;
+	find_bsp(mb, &bsp);
+	bsp_set_location(&bsp);
+
+	uintptr_t reserve_end = (uintptr_t)__pend;
+	if (bsp.mod_base + bsp.mod_size > reserve_end) {
+		reserve_end = bsp.mod_base + bsp.mod_size;
+	}
+
+	early_vm_init(reserve_end);
 
 	e820_scan(PTR32(mb->mmap_addr), mb->mmap_length);
 
@@ -102,16 +129,20 @@ int ml_init(uintptr_t arg)
 	put_cpu(this_cpu);
 
 	struct vm_zone_descriptor vm_zones[] = {
-		{.zd_id = VM_ZONE_DMA,
-	         .zd_node = 0,
-	         .zd_name = "dma",
-	         .zd_base = 0x00,
-	         .zd_limit = 0xffffff},
-		{.zd_id = VM_ZONE_NORMAL,
-	         .zd_node = 0,
-	         .zd_name = "normal",
-	         .zd_base = 0x1000000,
-	         .zd_limit = UINTPTR_MAX},
+		{
+			.zd_id = VM_ZONE_DMA,
+			.zd_node = 0,
+			.zd_name = "dma",
+			.zd_base = 0x00,
+			.zd_limit = 0xffffff,
+		},
+		{
+			.zd_id = VM_ZONE_NORMAL,
+			.zd_node = 0,
+			.zd_name = "normal",
+			.zd_base = 0x1000000,
+			.zd_limit = UINTPTR_MAX,
+		},
 	};
 
 	vm_bootstrap(vm_zones, sizeof vm_zones / sizeof vm_zones[0]);

arch/x86_64/irq.c

@@ -1,4 +1,5 @@
 #include <arch/irq.h>
+#include <arch/msr.h>
 #include <arch/ports.h>
 #include <mango/cpu.h>
 #include <mango/libc/string.h>
@@ -6,9 +7,16 @@
 #include <mango/machine/irq.h>
 #include <mango/panic.h>
 #include <mango/sched.h>
+#include <mango/syscall.h>
 #include <stddef.h>
 
-#define MAX_ISR_HANDLERS 16
+#define MAX_ISR_HANDLERS  16
+
+#define PF_PRESENT        0x01u
+#define PF_WRITE          0x02u
+#define PF_USER           0x04u
+#define PF_RESERVED_WRITE 0x08u
+#define PF_IFETCH         0x10u
 
 extern void syscall_gate(void);
 extern uintptr_t pf_faultptr(void);
@@ -20,6 +28,27 @@ static struct idt idt;
 static int idt_initialised = 0;
 static uintptr_t int_entry_points[NR_IDT_ENTRIES];
 
+static void set_syscall_gate(uintptr_t rip)
+{
+	uint64_t user_cs = 0x13;
+	uint64_t kernel_cs = 0x8;
+
+	uintptr_t star_reg = 0xC0000081;
+	uintptr_t lstar_reg = 0xC0000082;
+	uintptr_t sfmask_reg = 0xC0000084;
+
+	uint64_t selectors = 0;
+	selectors |= (user_cs) << 48;
+	selectors |= (kernel_cs) << 32;
+
+	/* disable interrupts */
+	uint64_t flag_mask = 0x200;
+
+	wrmsr(star_reg, selectors);
+	wrmsr(lstar_reg, rip);
+	wrmsr(sfmask_reg, flag_mask);
+}
+
 static void set_idt_gate(
 	struct idt *idt,
 	uint8_t index,
@@ -39,7 +68,7 @@ static void set_idt_gate(
 	idt->i_entries[index].reserved = 0;
 }
 
-static void gpf_handler(struct cpu_context *regs)
+static void gpf_handler(struct ml_cpu_context *regs)
 {
 	int ext = regs->err_no & 1;
 	int table = (regs->err_no >> 1) & 0x03;
@@ -55,43 +84,37 @@ static void gpf_handler(struct cpu_context *regs)
 		regs->rip);
 }
 
-static void pf_handler(struct cpu_context *regs)
+static void pf_handler(struct ml_cpu_context *regs)
 {
+	enum pmap_fault_flags fault_flags = 0;
+
+	(regs->err_no & PF_PRESENT) && (fault_flags |= PMAP_FAULT_PRESENT);
+	(regs->err_no & PF_USER) && (fault_flags |= PMAP_FAULT_USER);
+	(regs->err_no & PF_WRITE) && (fault_flags |= PMAP_FAULT_WRITE);
+	(regs->err_no & PF_IFETCH) && (fault_flags |= PMAP_FAULT_IFETCH);
+	(regs->err_no & PF_RESERVED_WRITE)
+		&& (fault_flags |= PMAP_FAULT_BADCFG);
+
+	virt_addr_t fault_ptr = pf_faultptr();
+
+	kern_status_t status = KERN_FATAL_ERROR;
+
+	if (regs->err_no & PF_USER) {
+		status = pmap_handle_fault(fault_ptr, fault_flags);
+	}
+
+	if (status == KERN_OK) {
+		return;
+	}
+
 	panic_irq(
 		regs,
 		"page fault (%016llx %016llx %016llx)",
-		pf_faultptr(),
+		fault_ptr,
 		regs->rip,
 		regs->err_no);
 }
 
-#if 0
-static void set_syscall_gate(uintptr_t rip)
-{
-	/* sysret adds 0x10 to this to get cs, and 0x8 to get ss
-	 * note that the CPU should force the RPL to 3 when loading
-	 * the selector by using user_cs | 3. However, this doesn't happen
-	 * in certain scenarios (specifically, QEMU + KVM on a Ryzen 5 1600X). */
-	uint64_t user_cs = 0x13;
-	uint64_t kernel_cs = 0x8;
-
-	uintptr_t star_reg = 0xC0000081;
-	uintptr_t lstar_reg = 0xC0000082;
-	uintptr_t sfmask_reg = 0xC0000084;
-
-	uint64_t selectors = 0;
-	selectors |= (user_cs) << 48;
-	selectors |= (kernel_cs) << 32;
-
-	/* disable interrupts */
-	uint64_t flag_mask = 0x200;
-
-	write_msr(star_reg, selectors);
-	write_msr(lstar_reg, rip);
-	write_msr(sfmask_reg, flag_mask);
-}
-#endif
-
 static void init_pic(void)
 {
 	// Remap the PIC
@@ -129,6 +152,8 @@ int idt_init(struct idt_ptr *ptr)
 		init_global_idt();
 	}
 
+	set_syscall_gate((uintptr_t)syscall_gate);
+
 	ptr->i_limit = sizeof(idt) - 1;
 	ptr->i_base = (uintptr_t)&idt;
 
@@ -141,7 +166,7 @@ int idt_load(struct idt_ptr *ptr)
 	return 0;
 }
 
-void isr_dispatch(struct cpu_context *regs)
+void isr_dispatch(struct ml_cpu_context *regs)
 {
 	int_hook h = isr_handlers[regs->int_no];
 	if (h) {
@@ -160,7 +185,7 @@ void irq_ack(unsigned int vec)
 	outportb(0x20, 0x20);
 }
 
-void irq_dispatch(struct cpu_context *regs)
+void irq_dispatch(struct ml_cpu_context *regs)
 {
 	end_charge_period();
 
@@ -178,8 +203,38 @@ void irq_dispatch(struct cpu_context *regs)
 	start_charge_period();
 }
 
-void syscall_dispatch(struct cpu_context *regs)
+void syscall_dispatch(struct ml_cpu_context *regs)
 {
+	unsigned int sysid = regs->rax;
+	virt_addr_t syscall_impl = syscall_get_func(sysid);
+
+	if (syscall_impl == 0) {
+		regs->rax = KERN_UNSUPPORTED;
+		return;
+	}
+
+#define SYSCALL_SIGNATURE(...)                                                 \
+	intptr_t (*__VA_ARGS__)(                                               \
+		uintptr_t,                                                     \
+		uintptr_t,                                                     \
+		uintptr_t,                                                     \
+		uintptr_t,                                                     \
+		uintptr_t,                                                     \
+		uintptr_t,                                                     \
+		uintptr_t,                                                     \
+		uintptr_t)
+
+	SYSCALL_SIGNATURE(fn) = (SYSCALL_SIGNATURE())syscall_impl;
+
+	regs->rax
+		= fn(regs->rdi,
+	             regs->rsi,
+	             regs->rdx,
+	             regs->r12,
+	             regs->r8,
+	             regs->r9,
+	             regs->r13,
+	             regs->r14);
 }
 
 void hook_irq(enum irq_vector vec, struct irq_hook *hook)
@@ -194,263 +249,263 @@ void unhook_irq(enum irq_vector vec, struct irq_hook *hook)
 	queue_delete(hook_queue, &hook->irq_entry);
 }
 
-extern void _isr0();
-extern void _isr1();
-extern void _isr2();
-extern void _isr3();
-extern void _isr4();
-extern void _isr5();
-extern void _isr6();
-extern void _isr7();
-extern void _isr8();
-extern void _isr9();
-extern void _isr10();
-extern void _isr11();
-extern void _isr12();
-extern void _isr13();
-extern void _isr14();
-extern void _isr15();
-extern void _isr16();
-extern void _isr17();
-extern void _isr18();
-extern void _isr19();
-extern void _isr20();
-extern void _isr21();
-extern void _isr22();
-extern void _isr23();
-extern void _isr24();
-extern void _isr25();
-extern void _isr26();
-extern void _isr27();
-extern void _isr28();
-extern void _isr29();
-extern void _isr30();
-extern void _isr31();
+extern void _isr0(void);
+extern void _isr1(void);
+extern void _isr2(void);
+extern void _isr3(void);
+extern void _isr4(void);
+extern void _isr5(void);
+extern void _isr6(void);
+extern void _isr7(void);
+extern void _isr8(void);
+extern void _isr9(void);
+extern void _isr10(void);
+extern void _isr11(void);
+extern void _isr12(void);
+extern void _isr13(void);
+extern void _isr14(void);
+extern void _isr15(void);
+extern void _isr16(void);
+extern void _isr17(void);
+extern void _isr18(void);
+extern void _isr19(void);
+extern void _isr20(void);
+extern void _isr21(void);
+extern void _isr22(void);
+extern void _isr23(void);
+extern void _isr24(void);
+extern void _isr25(void);
+extern void _isr26(void);
+extern void _isr27(void);
+extern void _isr28(void);
+extern void _isr29(void);
+extern void _isr30(void);
+extern void _isr31(void);
 
-extern void _irq0();
-extern void _irq1();
-extern void _irq2();
-extern void _irq3();
-extern void _irq4();
-extern void _irq5();
-extern void _irq6();
-extern void _irq7();
-extern void _irq8();
-extern void _irq9();
-extern void _irq10();
-extern void _irq11();
-extern void _irq12();
-extern void _irq13();
-extern void _irq14();
-extern void _irq15();
-extern void _irq16();
-extern void _irq17();
-extern void _irq18();
-extern void _irq19();
-extern void _irq20();
-extern void _irq21();
-extern void _irq22();
-extern void _irq23();
-extern void _irq24();
-extern void _irq25();
-extern void _irq26();
-extern void _irq27();
-extern void _irq28();
-extern void _irq29();
-extern void _irq30();
-extern void _irq31();
-extern void _irq32();
-extern void _irq33();
-extern void _irq34();
-extern void _irq35();
-extern void _irq36();
-extern void _irq37();
-extern void _irq38();
-extern void _irq39();
-extern void _irq40();
-extern void _irq41();
-extern void _irq42();
-extern void _irq43();
-extern void _irq44();
-extern void _irq45();
-extern void _irq46();
-extern void _irq47();
-extern void _irq48();
-extern void _irq49();
-extern void _irq50();
-extern void _irq51();
-extern void _irq52();
-extern void _irq53();
-extern void _irq54();
-extern void _irq55();
-extern void _irq56();
-extern void _irq57();
-extern void _irq58();
-extern void _irq59();
-extern void _irq60();
-extern void _irq61();
-extern void _irq62();
-extern void _irq63();
-extern void _irq64();
-extern void _irq65();
-extern void _irq66();
-extern void _irq67();
-extern void _irq68();
-extern void _irq69();
-extern void _irq70();
-extern void _irq71();
-extern void _irq72();
-extern void _irq73();
-extern void _irq74();
-extern void _irq75();
-extern void _irq76();
-extern void _irq77();
-extern void _irq78();
-extern void _irq79();
-extern void _irq80();
-extern void _irq81();
-extern void _irq82();
-extern void _irq83();
-extern void _irq84();
-extern void _irq85();
-extern void _irq86();
-extern void _irq87();
-extern void _irq88();
-extern void _irq89();
-extern void _irq90();
-extern void _irq91();
-extern void _irq92();
-extern void _irq93();
-extern void _irq94();
-extern void _irq95();
-extern void _irq96();
-extern void _irq97();
-extern void _irq98();
-extern void _irq99();
-extern void _irq100();
-extern void _irq101();
-extern void _irq102();
-extern void _irq103();
-extern void _irq104();
-extern void _irq105();
-extern void _irq106();
-extern void _irq107();
-extern void _irq108();
-extern void _irq109();
-extern void _irq110();
-extern void _irq111();
-extern void _irq112();
-extern void _irq113();
-extern void _irq114();
-extern void _irq115();
-extern void _irq116();
-extern void _irq117();
-extern void _irq118();
-extern void _irq119();
-extern void _irq120();
-extern void _irq121();
-extern void _irq122();
-extern void _irq123();
-extern void _irq124();
-extern void _irq125();
-extern void _irq126();
-extern void _irq127();
-extern void _irq128();
-extern void _irq129();
-extern void _irq130();
-extern void _irq131();
-extern void _irq132();
-extern void _irq133();
-extern void _irq134();
-extern void _irq135();
-extern void _irq136();
-extern void _irq137();
-extern void _irq138();
-extern void _irq139();
-extern void _irq140();
-extern void _irq141();
-extern void _irq142();
-extern void _irq143();
-extern void _irq144();
-extern void _irq145();
-extern void _irq146();
-extern void _irq147();
-extern void _irq148();
-extern void _irq149();
-extern void _irq150();
-extern void _irq151();
-extern void _irq152();
-extern void _irq153();
-extern void _irq154();
-extern void _irq155();
-extern void _irq156();
-extern void _irq157();
-extern void _irq158();
-extern void _irq159();
-extern void _irq160();
-extern void _irq161();
-extern void _irq162();
-extern void _irq163();
-extern void _irq164();
-extern void _irq165();
-extern void _irq166();
-extern void _irq167();
-extern void _irq168();
-extern void _irq169();
-extern void _irq170();
-extern void _irq171();
-extern void _irq172();
-extern void _irq173();
-extern void _irq174();
-extern void _irq175();
-extern void _irq176();
-extern void _irq177();
-extern void _irq178();
-extern void _irq179();
-extern void _irq180();
-extern void _irq181();
-extern void _irq182();
-extern void _irq183();
-extern void _irq184();
-extern void _irq185();
-extern void _irq186();
-extern void _irq187();
-extern void _irq188();
-extern void _irq189();
-extern void _irq190();
-extern void _irq191();
-extern void _irq192();
-extern void _irq193();
-extern void _irq194();
-extern void _irq195();
-extern void _irq196();
-extern void _irq197();
-extern void _irq198();
-extern void _irq199();
-extern void _irq200();
-extern void _irq201();
-extern void _irq202();
-extern void _irq203();
-extern void _irq204();
-extern void _irq205();
-extern void _irq206();
-extern void _irq207();
-extern void _irq208();
-extern void _irq209();
-extern void _irq210();
-extern void _irq211();
-extern void _irq212();
-extern void _irq213();
-extern void _irq214();
-extern void _irq215();
-extern void _irq216();
-extern void _irq217();
-extern void _irq218();
-extern void _irq219();
-extern void _irq220();
-extern void _irq221();
-extern void _irq222();
-extern void _irq223();
+extern void _irq0(void);
+extern void _irq1(void);
+extern void _irq2(void);
+extern void _irq3(void);
+extern void _irq4(void);
+extern void _irq5(void);
+extern void _irq6(void);
+extern void _irq7(void);
+extern void _irq8(void);
+extern void _irq9(void);
+extern void _irq10(void);
+extern void _irq11(void);
+extern void _irq12(void);
+extern void _irq13(void);
+extern void _irq14(void);
+extern void _irq15(void);
+extern void _irq16(void);
+extern void _irq17(void);
+extern void _irq18(void);
+extern void _irq19(void);
+extern void _irq20(void);
+extern void _irq21(void);
+extern void _irq22(void);
+extern void _irq23(void);
+extern void _irq24(void);
+extern void _irq25(void);
+extern void _irq26(void);
+extern void _irq27(void);
+extern void _irq28(void);
+extern void _irq29(void);
+extern void _irq30(void);
+extern void _irq31(void);
+extern void _irq32(void);
+extern void _irq33(void);
+extern void _irq34(void);
+extern void _irq35(void);
+extern void _irq36(void);
+extern void _irq37(void);
+extern void _irq38(void);
+extern void _irq39(void);
+extern void _irq40(void);
+extern void _irq41(void);
+extern void _irq42(void);
+extern void _irq43(void);
+extern void _irq44(void);
+extern void _irq45(void);
+extern void _irq46(void);
+extern void _irq47(void);
+extern void _irq48(void);
+extern void _irq49(void);
+extern void _irq50(void);
+extern void _irq51(void);
+extern void _irq52(void);
+extern void _irq53(void);
+extern void _irq54(void);
+extern void _irq55(void);
+extern void _irq56(void);
+extern void _irq57(void);
+extern void _irq58(void);
+extern void _irq59(void);
+extern void _irq60(void);
+extern void _irq61(void);
+extern void _irq62(void);
+extern void _irq63(void);
+extern void _irq64(void);
+extern void _irq65(void);
+extern void _irq66(void);
+extern void _irq67(void);
+extern void _irq68(void);
+extern void _irq69(void);
+extern void _irq70(void);
+extern void _irq71(void);
+extern void _irq72(void);
+extern void _irq73(void);
+extern void _irq74(void);
+extern void _irq75(void);
+extern void _irq76(void);
+extern void _irq77(void);
+extern void _irq78(void);
+extern void _irq79(void);
+extern void _irq80(void);
+extern void _irq81(void);
+extern void _irq82(void);
+extern void _irq83(void);
+extern void _irq84(void);
+extern void _irq85(void);
+extern void _irq86(void);
+extern void _irq87(void);
+extern void _irq88(void);
+extern void _irq89(void);
+extern void _irq90(void);
+extern void _irq91(void);
+extern void _irq92(void);
+extern void _irq93(void);
+extern void _irq94(void);
+extern void _irq95(void);
+extern void _irq96(void);
+extern void _irq97(void);
+extern void _irq98(void);
+extern void _irq99(void);
+extern void _irq100(void);
+extern void _irq101(void);
+extern void _irq102(void);
+extern void _irq103(void);
+extern void _irq104(void);
+extern void _irq105(void);
+extern void _irq106(void);
+extern void _irq107(void);
+extern void _irq108(void);
+extern void _irq109(void);
+extern void _irq110(void);
+extern void _irq111(void);
+extern void _irq112(void);
+extern void _irq113(void);
+extern void _irq114(void);
+extern void _irq115(void);
+extern void _irq116(void);
+extern void _irq117(void);
+extern void _irq118(void);
+extern void _irq119(void);
+extern void _irq120(void);
+extern void _irq121(void);
+extern void _irq122(void);
+extern void _irq123(void);
+extern void _irq124(void);
+extern void _irq125(void);
+extern void _irq126(void);
+extern void _irq127(void);
+extern void _irq128(void);
+extern void _irq129(void);
+extern void _irq130(void);
+extern void _irq131(void);
+extern void _irq132(void);
+extern void _irq133(void);
+extern void _irq134(void);
+extern void _irq135(void);
+extern void _irq136(void);
+extern void _irq137(void);
+extern void _irq138(void);
+extern void _irq139(void);
+extern void _irq140(void);
+extern void _irq141(void);
+extern void _irq142(void);
+extern void _irq143(void);
+extern void _irq144(void);
+extern void _irq145(void);
+extern void _irq146(void);
+extern void _irq147(void);
+extern void _irq148(void);
+extern void _irq149(void);
+extern void _irq150(void);
+extern void _irq151(void);
+extern void _irq152(void);
+extern void _irq153(void);
+extern void _irq154(void);
+extern void _irq155(void);
+extern void _irq156(void);
+extern void _irq157(void);
+extern void _irq158(void);
+extern void _irq159(void);
+extern void _irq160(void);
+extern void _irq161(void);
+extern void _irq162(void);
+extern void _irq163(void);
+extern void _irq164(void);
+extern void _irq165(void);
+extern void _irq166(void);
+extern void _irq167(void);
+extern void _irq168(void);
+extern void _irq169(void);
+extern void _irq170(void);
+extern void _irq171(void);
+extern void _irq172(void);
+extern void _irq173(void);
+extern void _irq174(void);
+extern void _irq175(void);
+extern void _irq176(void);
+extern void _irq177(void);
+extern void _irq178(void);
+extern void _irq179(void);
+extern void _irq180(void);
+extern void _irq181(void);
+extern void _irq182(void);
+extern void _irq183(void);
+extern void _irq184(void);
+extern void _irq185(void);
+extern void _irq186(void);
+extern void _irq187(void);
+extern void _irq188(void);
+extern void _irq189(void);
+extern void _irq190(void);
+extern void _irq191(void);
+extern void _irq192(void);
+extern void _irq193(void);
+extern void _irq194(void);
+extern void _irq195(void);
+extern void _irq196(void);
+extern void _irq197(void);
+extern void _irq198(void);
+extern void _irq199(void);
+extern void _irq200(void);
+extern void _irq201(void);
+extern void _irq202(void);
+extern void _irq203(void);
+extern void _irq204(void);
+extern void _irq205(void);
+extern void _irq206(void);
+extern void _irq207(void);
+extern void _irq208(void);
+extern void _irq209(void);
+extern void _irq210(void);
+extern void _irq211(void);
+extern void _irq212(void);
+extern void _irq213(void);
+extern void _irq214(void);
+extern void _irq215(void);
+extern void _irq216(void);
+extern void _irq217(void);
+extern void _irq218(void);
+extern void _irq219(void);
+extern void _irq220(void);
+extern void _irq221(void);
+extern void _irq222(void);
+extern void _irq223(void);
 
 static uintptr_t int_entry_points[NR_IDT_ENTRIES] = {
 	[0] = (uintptr_t)_isr0,     [1] = (uintptr_t)_isr1,

arch/x86_64/panic.c

@@ -1,24 +1,25 @@
-#include "mango/machine/panic.h"
-#include "mango/vm.h"
-#include <mango/printk.h>
-#include <mango/libc/stdio.h>
 #include <arch/irq.h>
+#include <mango/libc/stdio.h>
+#include <mango/machine/cpu.h>
+#include <mango/machine/panic.h>
+#include <mango/printk.h>
+#include <mango/vm.h>
 
-#define R_CF 0
-#define R_PF 2
-#define R_AF 4
-#define R_ZF 6
-#define R_SF 7
-#define R_TF 8
-#define R_IF 9
-#define R_DF 10
-#define R_OF 11
-#define R_NT 14
-#define R_VM 17
-#define R_AC 18
+#define R_CF  0
+#define R_PF  2
+#define R_AF  4
+#define R_ZF  6
+#define R_SF  7
+#define R_TF  8
+#define R_IF  9
+#define R_DF  10
+#define R_OF  11
+#define R_NT  14
+#define R_VM  17
+#define R_AC  18
 #define R_VIF 19
 #define R_VIP 20
-#define R_ID 21
+#define R_ID  21
 #define R_MAX 21
 
 struct stack_frame {
@@ -82,7 +83,11 @@ static void print_rflags(uintptr_t rflags)
 		if (rflags & (1 << i)) {
 			const char *name = pf_rfl_name(i);
 			if (name) {
-				buf_i += snprintf(buf + buf_i, sizeof(buf) - buf_i, " %s", name);
+				buf_i += snprintf(
+					buf + buf_i,
+					sizeof(buf) - buf_i,
+					" %s",
+					name);
 			}
 		}
 	}
@@ -91,30 +96,43 @@ static void print_rflags(uintptr_t rflags)
 	printk(buf);
 }
 
-void ml_print_cpu_state(struct cpu_context *ctx)
+void ml_print_cpu_state(struct ml_cpu_context *ctx)
 {
 	printk("cpu state:");
 	if (ctx) {
 		printk("  rax %016llx rbx %016llx rcx %016llx",
-		       ctx->rax, ctx->rbx, ctx->rcx);
+		       ctx->rax,
+		       ctx->rbx,
+		       ctx->rcx);
 		printk("  rdx %016llx rsi %016llx rdi %016llx",
-		       ctx->rdx, ctx->rsi, ctx->rdi);
+		       ctx->rdx,
+		       ctx->rsi,
+		       ctx->rdi);
 		printk("  rsp %016llx rbp %016llx r8  %016llx",
-		       ctx->rsp, ctx->rbp, ctx->r8);
+		       ctx->rsp,
+		       ctx->rbp,
+		       ctx->r8);
 		printk("  r9  %016llx r10 %016llx r11 %016llx",
-		       ctx->r9, ctx->r10, ctx->r11);
+		       ctx->r9,
+		       ctx->r10,
+		       ctx->r11);
 		printk("  r12 %016llx r13 %016llx r14 %016llx",
-		       ctx->r12, ctx->r13, ctx->r14);
+		       ctx->r12,
+		       ctx->r13,
+		       ctx->r14);
 		printk("  r15 %016llx rip %016llx cs  %04x ss %04x",
-		       ctx->r15, ctx->rip, ctx->cs, ctx->ss);
+		       ctx->r15,
+		       ctx->rip,
+		       ctx->cs,
+		       ctx->ss);
 		print_rflags(ctx->rflags);
 	}
 
 	uintptr_t cr0 = 0, cr2 = 0, cr3 = 0, cr4 = 0;
-	asm volatile("mov %%cr0, %%rax" : "=a" (cr0));
-	asm volatile("mov %%cr2, %%rax" : "=a" (cr2));
-	asm volatile("mov %%cr3, %%rax" : "=a" (cr3));
-	asm volatile("mov %%cr4, %%rax" : "=a" (cr4));
+	asm volatile("mov %%cr0, %%rax" : "=a"(cr0));
+	asm volatile("mov %%cr2, %%rax" : "=a"(cr2));
+	asm volatile("mov %%cr3, %%rax" : "=a"(cr3));
+	asm volatile("mov %%cr4, %%rax" : "=a"(cr4));
 	printk("  cr0 %016llx cr2 %016llx", cr0, cr2);
 	printk("  cr3 %016llx cr4 %016llx", cr3, cr4);
 }
@@ -135,7 +153,12 @@ static void print_stack_item(uintptr_t addr)
 	int found = -1;
 
 	if (found == 0 && name[0] != '\0') {
-		i += snprintf(buf + i, sizeof(buf) - i, "%s+0x%lx", name, offset);
+		i += snprintf(
+			buf + i,
+			sizeof(buf) - i,
+			"%s+0x%lx",
+			name,
+			offset);
 	} else {
 		i += snprintf(buf + i, sizeof(buf) - i, "?");
 	}
@@ -152,9 +175,8 @@ static void print_stack_trace(uintptr_t ip, uintptr_t *bp)
 
 	int max_frames = 10, current_frame = 0;
 	while (1) {
-		if (!vm_virt_to_phys(stk) ||
-		    bp == NULL ||
-		    current_frame > max_frames) {
+		if (!vm_virt_to_phys(stk) || bp == NULL
+		    || current_frame > max_frames) {
 			break;
 		}
 
@@ -169,11 +191,11 @@ static void print_stack_trace(uintptr_t ip, uintptr_t *bp)
 void ml_print_stack_trace(uintptr_t ip)
 {
 	uintptr_t *bp;
-	asm volatile("mov %%rbp, %0" : "=r" (bp));
+	asm volatile("mov %%rbp, %0" : "=r"(bp));
 	print_stack_trace(ip, bp);
 }
 
-void ml_print_stack_trace_irq(struct cpu_context *ctx)
+void ml_print_stack_trace_irq(struct ml_cpu_context *ctx)
 {
 	print_stack_trace(ctx->rip, (uintptr_t *)ctx->rbp);
 }

arch/x86_64/pmap.c

@@ -1,9 +1,13 @@
 #include <mango/compiler.h>
+#include <mango/libc/stdio.h>
 #include <mango/memblock.h>
 #include <mango/pmap.h>
 #include <mango/printk.h>
+#include <mango/sched.h>
 #include <mango/status.h>
 #include <mango/types.h>
+#include <mango/vm-object.h>
+#include <mango/vm-region.h>
 #include <mango/vm.h>
 
 /* some helpful datasize constants */
@@ -11,7 +15,7 @@
 #define C_2GiB          (2 * C_1GiB)
 
 #define BAD_INDEX       ((unsigned int)-1)
-#define PTR_TO_ENTRY(x) (((x) & ~VM_PAGE_MASK) | PTE_PRESENT | PTE_RW)
+#define PTR_TO_ENTRY(x) (((x) & ~VM_PAGE_MASK) | PTE_PRESENT | PTE_RW | PTE_USR)
 #define ENTRY_TO_PTR(x) ((x) & ~VM_PAGE_MASK)
 
 #define PFN(x)          ((x) >> VM_PAGE_SHIFT)
@@ -133,12 +137,11 @@ static void delete_pdir(phys_addr_t pd)
 
 static kern_status_t do_pmap_add(
 	pmap_t pmap,
-	void *p,
+	virt_addr_t pv,
 	pfn_t pfn,
 	enum vm_prot prot,
 	enum page_size size)
 {
-	uintptr_t pv = (uintptr_t)p;
 	unsigned int pml4t_index = BAD_INDEX, pdpt_index = BAD_INDEX,
 		     pd_index = BAD_INDEX, pt_index = BAD_INDEX;
 
@@ -261,11 +264,11 @@ void pmap_bootstrap(void)
 	/* map 2GiB at the end of the address space to
 	   replace the mapping created by start_32 and allow access to
 	   the kernel and memblock-allocated data. */
-	uintptr_t vbase = VM_KERNEL_VOFFSET;
+	virt_addr_t vbase = VM_KERNEL_VOFFSET;
 	for (size_t i = 0; i < C_2GiB; i += hugepage_sz) {
 		do_pmap_add(
 			kernel_pmap,
-			(void *)(vbase + i),
+			vbase + i,
 			PFN(i),
 			VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXEC
 				| VM_PROT_SVR,
@@ -285,7 +288,7 @@ void pmap_bootstrap(void)
 	for (size_t i = 0; i < pmem_limit; i += hugepage_sz) {
 		do_pmap_add(
 			kernel_pmap,
-			(void *)(vbase + i),
+			vbase + i,
 			PFN(i),
 			VM_PROT_READ | VM_PROT_WRITE | VM_PROT_SVR
 				| VM_PROT_NOCACHE,
@@ -297,16 +300,76 @@ void pmap_bootstrap(void)
 
 pmap_t pmap_create(void)
 {
-	return 0;
+	pmap_t pmap = alloc_pmap();
+	pmap_t kernel_pmap = get_kernel_pmap();
+
+	struct pml4t *pml4t = vm_phys_to_virt(pmap);
+	struct pml4t *kernel_pml4t = vm_phys_to_virt(kernel_pmap);
+
+	for (unsigned int i = 256; i < 512; i++) {
+		pml4t->p_entries[i] = kernel_pml4t->p_entries[i];
+	}
+
+	return pmap;
 }
 
 void pmap_destroy(pmap_t pmap)
 {
 }
 
+static void log_fault(virt_addr_t fault_addr, enum pmap_fault_flags flags)
+{
+	char flag_str[128] = {0};
+	size_t p = 0;
+
+	if (flags & PMAP_FAULT_PRESENT) {
+		p += snprintf(flag_str + p, sizeof flag_str - p, " PRESENT");
+	} else {
+		p += snprintf(flag_str + p, sizeof flag_str - p, " MISSING");
+	}
+
+	if (flags & PMAP_FAULT_USER) {
+		p += snprintf(flag_str + p, sizeof flag_str - p, " USER");
+	} else {
+		p += snprintf(flag_str + p, sizeof flag_str - p, " SVR");
+	}
+
+	if (flags & PMAP_FAULT_WRITE) {
+		p += snprintf(flag_str + p, sizeof flag_str - p, " WRITE");
+	} else {
+		p += snprintf(flag_str + p, sizeof flag_str - p, " READ");
+	}
+
+	if (flags & PMAP_FAULT_IFETCH) {
+		p += snprintf(flag_str + p, sizeof flag_str - p, " IFETCH");
+	}
+
+	if (flags & PMAP_FAULT_BADCFG) {
+		p += snprintf(flag_str + p, sizeof flag_str - p, " BADCFG");
+	}
+
+	printk("pmap: fault at 0x%llx (%s)", fault_addr, flag_str);
+}
+
+kern_status_t pmap_handle_fault(
+	virt_addr_t fault_addr,
+	enum pmap_fault_flags flags)
+{
+	// log_fault(fault_addr, flags);
+
+	if (flags & PMAP_FAULT_PRESENT) {
+		return KERN_FATAL_ERROR;
+	}
+
+	struct task *task = current_task();
+	struct vm_region *space = task->t_address_space;
+
+	return vm_region_demand_map(space, fault_addr, flags);
+}
+
 kern_status_t pmap_add(
 	pmap_t pmap,
-	void *p,
+	virt_addr_t p,
 	pfn_t pfn,
 	enum vm_prot prot,
 	enum pmap_flags flags)
@@ -321,7 +384,7 @@ kern_status_t pmap_add(
 
 kern_status_t pmap_add_block(
 	pmap_t pmap,
-	void *p,
+	virt_addr_t p,
 	pfn_t pfn,
 	size_t len,
 	enum vm_prot prot,

arch/x86_64/thread.c

@@ -1,12 +1,16 @@
+#include <mango/machine/cpu.h>
 #include <mango/machine/thread.h>
 
+/* this is the context information restored by ml_thread_switch.
+ * since ml_thread_switch only jumps to kernel-mode, IRETQ isn't used,
+ * and the extra register values needed by IRETQ aren't present. */
 struct thread_ctx {
 	uint64_t r15, r14, r13, r12, r11, r10, r9, r8;
 	uint64_t rdi, rsi, rbp, unused_rsp, rbx, rdx, rcx, rax;
 	uint64_t rfl;
 } __packed;
 
-void prepare_stack(uintptr_t ip, uintptr_t *sp)
+void ml_thread_prepare_kernel_context(uintptr_t ip, uintptr_t *sp)
 {
 	(*sp) -= sizeof(uintptr_t);
 	uintptr_t *dest_ip = (uintptr_t *)(*sp);
@@ -18,3 +22,20 @@ void prepare_stack(uintptr_t ip, uintptr_t *sp)
 
 	ctx->rfl = 0x202;
 }
+
+extern void ml_thread_prepare_user_context(
+	virt_addr_t ip,
+	virt_addr_t user_sp,
+	virt_addr_t *kernel_sp)
+{
+	(*kernel_sp) -= sizeof(struct ml_cpu_context);
+
+	struct ml_cpu_context *ctx = (struct ml_cpu_context *)(*kernel_sp);
+	ctx->rip = ip;
+	ctx->rsp = user_sp;
+	ctx->ss = 0x23;
+	ctx->cs = 0x1B;
+	ctx->rflags = 0x202;
+	ctx->rdi = 0;   // arg 0
+	ctx->rsi = 0;   // arg 1
+}

arch/x86_64/thread_switch.S

@@ -1,13 +1,13 @@
-	.code64
+.code64
 
-	.extern THREAD_sp
+.extern THREAD_sp
 
-	.global switch_to
-	.type switch_to, @function
+.global ml_thread_switch
+.type ml_thread_switch, @function
 
 	// %rdi = (struct thread *) current thread.
 	// %rsi = (struct thread *) next thread.
-switch_to:
+ml_thread_switch:
 	pushfq
 
 	push %rax
@@ -50,3 +50,27 @@ switch_to:
 	popfq
 
 	ret
+
+
+.global ml_thread_switch_user
+.type ml_thread_switch_user, @function
+ml_thread_switch_user:
+	pop %r15
+	pop %r14
+	pop %r13
+	pop %r12
+	pop %r11
+	pop %r10
+	pop %r9
+	pop %r8
+	pop %rdi
+	pop %rsi
+	pop %rbp
+	add $8, %rsp
+	pop %rbx
+	pop %rdx
+	pop %rcx
+	pop %rax
+
+	add $16, %rsp
+	iretq

arch/x86_64/tss.c

@@ -0,0 +1,48 @@
+#include "arch/msr.h"
+
+#include <arch/gdt.h>
+#include <arch/tss.h>
+#include <mango/libc/string.h>
+
+static void tss_flush(int index)
+{
+	index *= sizeof(struct gdt_entry);
+	index |= 3;
+	asm volatile("mov %0, %%eax; ltr %%ax" ::"r"(index));
+}
+
+void tss_init(struct tss *tss, struct tss_ptr *ptr)
+{
+	memset(tss, 0x0, sizeof *tss);
+
+	ptr->tss_base = (uint64_t)tss;
+	ptr->tss_limit = (uint16_t)sizeof *tss;
+}
+
+void tss_load(struct tss *tss)
+{
+	tss_flush(TSS_GDT_INDEX);
+
+	uintptr_t kernel_gs_base_reg = 0xC0000102;
+	wrmsr(kernel_gs_base_reg, (uintptr_t)tss);
+}
+
+virt_addr_t tss_get_kstack(struct tss *tss)
+{
+	return tss->rsp0;
+}
+
+virt_addr_t tss_get_ustack(struct tss *tss)
+{
+	return tss->rsp2;
+}
+
+void tss_set_kstack(struct tss *tss, virt_addr_t sp)
+{
+	tss->rsp0 = sp;
+}
+
+void tss_set_ustack(struct tss *tss, virt_addr_t sp)
+{
+	tss->rsp2 = sp;
+}

ds/bitmap.c

@@ -1,5 +1,5 @@
-#include <mango/libc/string.h>
 #include <mango/bitmap.h>
+#include <mango/libc/string.h>
 
 void bitmap_zero(unsigned long *map, unsigned long nbits)
 {
@@ -25,7 +25,7 @@ void bitmap_set(unsigned long *map, unsigned long bit)
 void bitmap_clear(unsigned long *map, unsigned long bit)
 {
 	unsigned long index = bit / BITS_PER_WORD;
-	unsigned long offset = bit & (BITS_PER_WORD - 1);
+	unsigned long offset = (BITS_PER_WORD - bit - 1) & (BITS_PER_WORD - 1);
 	unsigned long mask = 1ul << offset;
 
 	map[index] &= ~mask;
@@ -38,7 +38,6 @@ bool bitmap_check(unsigned long *map, unsigned long bit)
 	unsigned long mask = 1ul << offset;
 
 	return (map[index] & mask) != 0 ? true : false;
-
 }
 
 unsigned int bitmap_count_set(unsigned long *map, unsigned long nbits)

include/mango/bsp.h

@@ -0,0 +1,38 @@
+#ifndef MANGO_BSP_H_
+#define MANGO_BSP_H_
+
+#include <mango/compiler.h>
+#include <mango/status.h>
+#include <mango/types.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#define BSP_MAGIC 0xcafebabe
+
+struct task;
+
+struct bsp_trailer {
+	/* these fields are stored in big endian in the package itself */
+	uint32_t bsp_magic;
+	uint64_t bsp_fs_offset;
+	uint32_t bsp_fs_len;
+	uint64_t bsp_exec_offset;
+	uint32_t bsp_exec_len;
+	uint64_t bsp_text_faddr, bsp_text_vaddr, bsp_text_size;
+	uint64_t bsp_data_faddr, bsp_data_vaddr, bsp_data_size;
+	uint64_t bsp_exec_entry;
+} __packed;
+
+struct bsp {
+	/* the values in this struct are stored in host byte order */
+	struct bsp_trailer bsp_trailer;
+	struct vm_object *bsp_vmo;
+};
+
+extern void bsp_set_location(const struct boot_module *mod);
+extern void bsp_get_location(struct boot_module *out);
+
+extern kern_status_t bsp_load(struct bsp *bsp, const struct boot_module *mod);
+extern kern_status_t bsp_launch_async(struct bsp *bsp, struct task *task);
+
+#endif

include/mango/handle.h

@@ -0,0 +1,56 @@
+#ifndef MANGO_HANDLE_H_
+#define MANGO_HANDLE_H_
+
+#include <mango/bitmap.h>
+#include <mango/status.h>
+#include <stdint.h>
+
+/* subtract 32 bytes to account for the handle bitmap */
+#define HANDLES_PER_TABLE ((4096 - 32) / sizeof(struct handle))
+#define REFS_PER_TABLE    ((4096 - 64) / sizeof(struct handle_table *))
+
+typedef uint32_t kern_handle_t;
+
+struct object;
+
+struct handle {
+	union {
+		struct object *h_object;
+		uint64_t __x;
+	};
+
+	uint64_t h_flags;
+};
+
+struct handle_table {
+	union {
+		struct {
+			/* bitmap tracks which bits in t_handle_list are
+			 * allocated */
+			DECLARE_BITMAP(t_handle_map, HANDLES_PER_TABLE);
+			struct handle t_handle_list[HANDLES_PER_TABLE];
+		} t_handles;
+
+		struct {
+			/* bitmap tracks which sub-tables are fully-allocated */
+			DECLARE_BITMAP(t_subtable_map, REFS_PER_TABLE);
+			struct handle_table *t_subtable_list[REFS_PER_TABLE];
+		} t_subtables;
+	};
+};
+
+extern struct handle_table *handle_table_create(void);
+extern void handle_table_destroy(struct handle_table *tab);
+
+extern kern_status_t handle_table_alloc_handle(
+	struct handle_table *tab,
+	struct handle **out_slot,
+	kern_handle_t *out_handle);
+extern void handle_table_free_handle(
+	struct handle_table *tab,
+	kern_handle_t handle);
+extern struct handle *handle_table_get_handle(
+	struct handle_table *tab,
+	kern_handle_t handle);
+
+#endif

include/mango/memblock.h

@@ -22,21 +22,22 @@
 #ifndef MANGO_MEMBLOCK_H_
 #define MANGO_MEMBLOCK_H_
 
-#include <stddef.h>
 #include <limits.h>
 #include <mango/types.h>
+#include <stddef.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#define MEMBLOCK_INIT_MEMORY_REGION_COUNT 128
+#define MEMBLOCK_INIT_MEMORY_REGION_COUNT   128
 #define MEMBLOCK_INIT_RESERVED_REGION_COUNT 128
 
-#define __for_each_mem_range(i, type_a, type_b, p_start, p_end) \
-	for ((i)->__idx = 0, __next_memory_region(i, type_a, type_b, p_start, p_end); \
-			(i)->__idx != ULLONG_MAX; \
-			__next_memory_region(i, type_a, type_b, p_start, p_end))
+#define __for_each_mem_range(i, type_a, type_b, p_start, p_end)                \
+	for ((i)->__idx = 0,                                                   \
+	    __next_memory_region(i, type_a, type_b, p_start, p_end);           \
+	     (i)->__idx != ULLONG_MAX;                                         \
+	     __next_memory_region(i, type_a, type_b, p_start, p_end))
 
 /* iterate through all memory regions known to memblock.
 
@@ -47,7 +48,7 @@ extern "C" {
 
    @param i the iterator. this should be a pointer of type struct memblock_iter.
             for each iteration, this structure will be filled with details about
-	    the current memory region.
+            the current memory region.
    @param p_start the lower bound of the memory region to iterate through.
             if you don't want to use a lower bound, pass 0.
    @param p_end the upper bound of the memory region to iterate through.
@@ -65,7 +66,7 @@ extern "C" {
      struct memblock_iter it;
      for_each_mem_region (&it, 0x40000, 0x80000) { ... }
 */
-#define for_each_mem_range(i, p_start, p_end) \
+#define for_each_mem_range(i, p_start, p_end)                                  \
 	__for_each_mem_range(i, &memblock.memory, NULL, p_start, p_end)
 
 /* iterate through all memory regions reserved using memblock.
@@ -77,7 +78,7 @@ extern "C" {
 
    @param i the iterator. this should be a pointer of type struct memblock_iter.
             for each iteration, this structure will be filled with details about
-	    the current memory region.
+            the current memory region.
    @param p_start the lower bound of the memory region to iterate through.
             if you don't want to use a lower bound, pass 0.
    @param p_end the upper bound of the memory region to iterate through.
@@ -95,7 +96,7 @@ extern "C" {
      struct memblock_iter it;
      for_each_reserved_mem_region (&it, 0x40000, 0x80000) { ... }
 */
-#define for_each_reserved_mem_range(i, p_start, p_end) \
+#define for_each_reserved_mem_range(i, p_start, p_end)                         \
 	__for_each_mem_range(i, &memblock.reserved, NULL, p_start, p_end)
 
 /* iterate through all memory regions known by memblock to be free.
@@ -108,7 +109,7 @@ extern "C" {
 
    @param i the iterator. this should be a pointer of type struct memblock_iter.
             for each iteration, this structure will be filled with details about
-	    the current memory region.
+            the current memory region.
    @param p_start the lower bound of the memory region to iterate through.
             if you don't want to use a lower bound, pass 0.
    @param p_end the upper bound of the memory region to iterate through.
@@ -138,19 +139,25 @@ extern "C" {
      - 0x08000 -> 0x08fff
      - 0x10000 -> 0x1ffff
 */
-#define for_each_free_mem_range(i, p_start, p_end) \
-	__for_each_mem_range(i, &memblock.memory, &memblock.reserved, p_start, p_end)
+#define for_each_free_mem_range(i, p_start, p_end)                             \
+	__for_each_mem_range(                                                  \
+		i,                                                             \
+		&memblock.memory,                                              \
+		&memblock.reserved,                                            \
+		p_start,                                                       \
+		p_end)
 
 typedef uint64_t memblock_index_t;
 
 enum memblock_region_status {
-	/* Used in memblock.memory regions, indicates that the memory region exists */
+	/* Used in memblock.memory regions, indicates that the memory region
+	 * exists */
 	MEMBLOCK_MEMORY = 0,
-	/* Used in memblock.reserved regions, indicates that the memory region was reserved
-	 * by a call to memblock_alloc() */
+	/* Used in memblock.reserved regions, indicates that the memory region
+	 * was reserved by a call to memblock_alloc() */
 	MEMBLOCK_ALLOC,
-	/* Used in memblock.reserved regions, indicates that the memory region was reserved
-	 * by a call to memblock_reserve() */
+	/* Used in memblock.reserved regions, indicates that the memory region
+	 * was reserved by a call to memblock_reserve() */
 	MEMBLOCK_RESERVED,
 };
 
@@ -176,9 +183,10 @@ struct memblock {
 	/* bounds of the memory region that can be used by memblock_alloc()
 	   both of these are virtual addresses */
 	uintptr_t m_alloc_start, m_alloc_end;
-	/* memblock assumes that all memory in the alloc zone is contiguously mapped
-	   (if paging is enabled). m_voffset is the offset that needs to be added to
-	   a given physical address to get the corresponding virtual address */
+	/* memblock assumes that all memory in the alloc zone is contiguously
+	   mapped (if paging is enabled). m_voffset is the offset that needs to
+	   be added to a given physical address to get the corresponding virtual
+	   address */
 	uintptr_t m_voffset;
 
 	struct memblock_type memory;
@@ -212,7 +220,10 @@ extern int __next_mem_range(struct memblock_iter *it);
    @param voffset the offset between the physical address of a given page and
                   its corresponding virtual address.
 */
-extern int memblock_init(uintptr_t alloc_start, uintptr_t alloc_end, uintptr_t voffset);
+extern int memblock_init(
+	uintptr_t alloc_start,
+	uintptr_t alloc_end,
+	uintptr_t voffset);
 
 /* add a region of memory to memblock.
 
@@ -234,7 +245,8 @@ extern int memblock_add(phys_addr_t base, size_t size);
    reserved memory will not be used by memblock_alloc(), and will remain
    reserved when the vm_page memory map is initialised.
 
-   @param base the physical address of the start of the memory region to reserve.
+   @param base the physical address of the start of the memory region to
+   reserve.
    @oaram size the size of the memory region to reserve in bytes.
 */
 extern int memblock_reserve(phys_addr_t base, size_t size);
@@ -257,7 +269,7 @@ extern int memblock_reserve(phys_addr_t base, size_t size);
    @param size the size of the buffer to allocate in bytes.
    @param align the alignment to use. for example, an alignment of 4096
                 will result in the returned pointer being a multiple
-		of 4096. this must be a power of 2.
+                of 4096. this must be a power of 2.
 */
 extern void *memblock_alloc(size_t size, phys_addr_t align);
 
@@ -279,7 +291,7 @@ extern void *memblock_alloc(size_t size, phys_addr_t align);
    @param size the size of the buffer to allocate in bytes.
    @param align the alignment to use. for example, an alignment of 4096
                 will result in the returned pointer being a multiple
-		of 4096. this must be a power of 2.
+                of 4096. this must be a power of 2.
 */
 extern phys_addr_t memblock_alloc_phys(size_t size, phys_addr_t align);
 
@@ -310,7 +322,7 @@ extern int memblock_free_phys(phys_addr_t addr, size_t size);
 
    @param p the pointer to convert.
 */
-extern phys_addr_t memblock_virt_to_phys(void *p);
+extern phys_addr_t memblock_virt_to_phys(const void *p);
 
 /* convert a physical address returned by memblock
    to a virtual pointer.
@@ -319,9 +331,12 @@ extern phys_addr_t memblock_virt_to_phys(void *p);
 */
 extern void *memblock_phys_to_virt(phys_addr_t p);
 
-extern void __next_memory_region(struct memblock_iter *it, \
-		struct memblock_type *type_a, struct memblock_type *type_b,
-		phys_addr_t start, phys_addr_t end);
+extern void __next_memory_region(
+	struct memblock_iter *it,
+	struct memblock_type *type_a,
+	struct memblock_type *type_b,
+	phys_addr_t start,
+	phys_addr_t end);
 
 #ifdef __cplusplus
 }

include/mango/object.h

@@ -11,6 +11,28 @@
 extern "C" {
 #endif
 
+#define DEFINE_OBJECT_LOCK_FUNCTION(object_name, base)                         \
+	static inline void object_name##_lock(struct object_name *p)           \
+	{                                                                      \
+		object_lock(&p->base);                                         \
+	}                                                                      \
+	static inline void object_name##_unlock(struct object_name *p)         \
+	{                                                                      \
+		object_unlock(&p->base);                                       \
+	}                                                                      \
+	static inline void object_name##_lock_irqsave(                         \
+		struct object_name *p,                                         \
+		unsigned long *flags)                                          \
+	{                                                                      \
+		object_lock_irqsave(&p->base, flags);                          \
+	}                                                                      \
+	static inline void object_name##_unlock_irqrestore(                    \
+		struct object_name *p,                                         \
+		unsigned long flags)                                           \
+	{                                                                      \
+		object_unlock_irqrestore(&p->base, flags);                     \
+	}
+
 #define OBJECT_MAGIC    0xBADDCAFE
 #define OBJECT_NAME_MAX 64
 #define OBJECT_PATH_MAX 256
@@ -58,7 +80,9 @@ extern kern_status_t object_type_unregister(struct object_type *p);
 
 extern struct object *object_create(struct object_type *type);
 extern struct object *object_ref(struct object *obj);
-extern void object_deref(struct object *obj);
+extern void object_unref(struct object *obj);
+extern void object_add_handle(struct object *obj);
+extern void object_remove_handle(struct object *obj);
 extern void object_lock(struct object *obj);
 extern void object_unlock(struct object *obj);
 extern void object_lock_irqsave(struct object *obj, unsigned long *flags);

include/mango/panic.h

@@ -3,10 +3,11 @@
 
 #include <mango/compiler.h>
 
-struct cpu_context;
+struct ml_cpu_context;
 
 #define panic(...) panic_irq(NULL, __VA_ARGS__)
 
-extern void __noreturn panic_irq(struct cpu_context *ctx, const char *fmt, ...);
+extern void __noreturn
+panic_irq(struct ml_cpu_context *ctx, const char *fmt, ...);
 
 #endif

include/mango/pmap.h

@@ -3,12 +3,13 @@
 
 /* all the functions declared in this file are defined in arch/xyz/pmap.c */
 
-#include <mango/vm.h>
-#include <mango/status.h>
 #include <mango/machine/pmap.h>
+#include <mango/status.h>
+#include <mango/vm.h>
 #include <stddef.h>
 
-#define PFN(x) ((x) >> VM_PAGE_SHIFT)
+#define PMAP_INVALID ML_PMAP_INVALID
+#define PFN(x)       ((x) >> VM_PAGE_SHIFT)
 
 #ifdef __cplusplus
 extern "C" {
@@ -17,6 +18,29 @@ extern "C" {
 typedef ml_pmap_t pmap_t;
 typedef ml_pfn_t pfn_t;
 
+enum pmap_fault_flags {
+	/* if set, the faulting page is present, and the fault is
+	 * protection-related.
+	 * if clear, the faulting page is missing, and the
+	 * fault is due to the missing page.
+	 */
+	PMAP_FAULT_PRESENT = 0x01u,
+	/* if set, the faulting page was accessed from user mode.
+	 * if clear, the faulting page was accessed from kernel mode.
+	 */
+	PMAP_FAULT_USER = 0x02u,
+	/* if set, the fault was caused by a write operation.
+	 * if clear, the faulting page was caused by a read operation.
+	 */
+	PMAP_FAULT_WRITE = 0x04u,
+	/* if set, the fault was caused while fetching an instruction from the
+	 * faulting page.
+	 */
+	PMAP_FAULT_IFETCH = 0x08u,
+	/* if set, the fault was caused by misconfigured page tables */
+	PMAP_FAULT_BADCFG = 0x10u,
+};
+
 enum pmap_flags {
 	PMAP_NORMAL = 0x00u,
 	PMAP_HUGEPAGE = 0x01u,
@@ -29,8 +53,23 @@ extern pmap_t pmap_create(void);
 extern void pmap_destroy(pmap_t pmap);
 extern void pmap_switch(pmap_t pmap);
 
-extern kern_status_t pmap_add(pmap_t pmap, void *p, pfn_t pfn, enum vm_prot prot, enum pmap_flags flags);
-extern kern_status_t pmap_add_block(pmap_t pmap, void *p, pfn_t pfn, size_t len, enum vm_prot prot, enum pmap_flags flags);
+extern kern_status_t pmap_handle_fault(
+	virt_addr_t fault_addr,
+	enum pmap_fault_flags flags);
+
+extern kern_status_t pmap_add(
+	pmap_t pmap,
+	virt_addr_t p,
+	pfn_t pfn,
+	enum vm_prot prot,
+	enum pmap_flags flags);
+extern kern_status_t pmap_add_block(
+	pmap_t pmap,
+	virt_addr_t p,
+	pfn_t pfn,
+	size_t len,
+	enum vm_prot prot,
+	enum pmap_flags flags);
 
 extern kern_status_t pmap_remove(pmap_t pmap, void *p);
 extern kern_status_t pmap_remove_range(pmap_t pmap, void *p, size_t len);

include/mango/printk.h

@@ -7,6 +7,12 @@
 extern "C" {
 #endif
 
+#ifdef TRACE
+#define tracek(...) printk(__VA_ARGS__)
+#else
+#define tracek(...)
+#endif
+
 extern void early_printk_init(struct console *con);
 extern int printk(const char *format, ...);
 

include/mango/sched.h

@@ -10,6 +10,7 @@
 
 #define TASK_NAME_MAX       64
 #define PRIO_MAX            32
+#define PID_MAX             99999
 #define THREAD_KSTACK_ORDER VM_PAGE_4K
 #define THREAD_MAX          65536
 
@@ -75,13 +76,18 @@ struct task {
 	struct object t_base;
 
 	struct task *t_parent;
-	unsigned int t_id;
+	long t_id;
 	enum task_state t_state;
 	char t_name[TASK_NAME_MAX];
 
 	pmap_t t_pmap;
+	struct vm_region *t_address_space;
+	struct handle_table *t_handles;
 
 	struct btree_node t_tasklist;
+	struct queue_entry t_child_entry;
+
+	size_t t_next_thread_id;
 	struct queue t_threads;
 	struct queue t_children;
 };
@@ -100,14 +106,16 @@ struct thread {
 	cycles_t tr_quantum_cycles, tr_quantum_target;
 	cycles_t tr_total_cycles;
 
-	uintptr_t tr_sp, tr_bp;
+	virt_addr_t tr_ip, tr_sp, tr_bp;
+	virt_addr_t tr_cpu_user_sp, tr_cpu_kernel_sp;
 
 	struct runqueue *tr_rq;
 
-	struct queue_entry tr_threads;
+	struct queue_entry tr_parent_entry;
 	struct queue_entry tr_rqentry;
 
 	struct vm_page *tr_kstack;
+	struct vm_object *tr_ustack;
 };
 
 struct runqueue {
@@ -175,15 +183,17 @@ extern void rq_remove_thread(struct runqueue *rq, struct thread *thr);
 extern struct runqueue *cpu_rq(unsigned int cpu);
 
 extern struct task *task_alloc(void);
+extern struct task *task_create(struct task *parent, const char *name);
 static inline struct task *task_ref(struct task *task)
 {
 	return OBJECT_CAST(struct task, t_base, object_ref(&task->t_base));
 }
-static inline void task_deref(struct task *task)
+static inline void task_unref(struct task *task)
 {
-	object_deref(&task->t_base);
+	object_unref(&task->t_base);
 }
 extern struct task *task_from_pid(unsigned int pid);
+extern struct thread *task_create_thread(struct task *parent);
 extern struct task *kernel_task(void);
 extern struct task *idle_task(void);
 extern cycles_t default_quantum(void);
@@ -198,20 +208,14 @@ extern void schedule_thread_on_cpu(struct thread *thr);
 extern void start_charge_period(void);
 extern void end_charge_period(void);
 
-static inline void task_lock_irqsave(struct task *task, unsigned long *flags)
-{
-	object_lock_irqsave(&task->t_base, flags);
-}
-
-static inline void task_unlock_irqrestore(
-	struct task *task,
-	unsigned long flags)
-{
-	object_unlock_irqrestore(&task->t_base, flags);
-}
+DEFINE_OBJECT_LOCK_FUNCTION(task, t_base)
 
 extern struct thread *thread_alloc(void);
-extern kern_status_t thread_init(struct thread *thr, uintptr_t ip);
+extern kern_status_t thread_init_kernel(struct thread *thr, virt_addr_t ip);
+extern kern_status_t thread_init_user(
+	struct thread *thr,
+	virt_addr_t ip,
+	virt_addr_t sp);
 extern int thread_priority(struct thread *thr);
 extern void idle(void);
 extern struct thread *create_kernel_thread(void (*fn)(void));

include/mango/status.h

@@ -15,6 +15,7 @@ typedef unsigned int kern_status_t;
 #define KERN_NO_DEVICE        (9)
 #define KERN_DEVICE_STUCK     (10)
 #define KERN_IO_ERROR         (11)
+#define KERN_FATAL_ERROR      (12)
 
 extern const char *kern_status_string(kern_status_t status);
 

include/mango/syscall.h

@@ -0,0 +1,20 @@
+#ifndef MANGO_SYSCALL_H_
+#define MANGO_SYSCALL_H_
+
+#include <mango/handle.h>
+#include <mango/status.h>
+#include <mango/vm.h>
+
+#define SYS_EXIT             1
+#define SYS_VM_OBJECT_CREATE 2
+
+extern kern_status_t sys_exit(int status);
+extern kern_status_t sys_vm_object_create(
+	const char *name,
+	size_t len,
+	enum vm_prot prot,
+	kern_handle_t *out_handle);
+
+extern virt_addr_t syscall_get_func(unsigned int sysid);
+
+#endif

include/mango/types.h

@@ -1,14 +1,22 @@
 #ifndef MANGO_TYPES_H_
 #define MANGO_TYPES_H_
 
+#include <stddef.h>
 #include <stdint.h>
 
 #define CYCLES_MAX UINT64_MAX
 
 typedef uintptr_t phys_addr_t;
+typedef uintptr_t virt_addr_t;
 typedef uint64_t cycles_t;
 typedef uint64_t sectors_t;
+typedef uint64_t off_t;
 
 typedef unsigned int umode_t;
 
+struct boot_module {
+	phys_addr_t mod_base;
+	size_t mod_size;
+};
+
 #endif

include/mango/util.h

@@ -1,27 +1,34 @@
 #ifndef MANGO_UTIL_H_
 #define MANGO_UTIL_H_
 
+#include <stdbool.h>
 #include <stddef.h>
 #include <stdint.h>
-#include <stdbool.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#define MIN(x, y) ((x) < (y) ? (x) : (y))
-#define MAX(x, y) ((x) > (y) ? (x) : (y))
+#define MIN(x, y)        ((x) < (y) ? (x) : (y))
+#define MAX(x, y)        ((x) > (y) ? (x) : (y))
 #define CLAMP(x, lo, hi) (MIN(MAX(x, lo), hi))
 
 extern uint64_t hash_string(const char *s);
 extern void data_size_to_string(size_t value, char *out, size_t outsz);
-static inline bool power_of_2(size_t x) { return (x > 0 && (x & (x - 1)) == 0); }
-static inline unsigned long long div64_pow2(unsigned long long x, unsigned long long y)
+static inline bool power_of_2(size_t x)
+{
+	return (x > 0 && (x & (x - 1)) == 0);
+}
+static inline unsigned long long div64_pow2(
+	unsigned long long x,
+	unsigned long long y)
 {
 	return x >> (__builtin_ctz(y));
 }
 
-static inline unsigned long long absdiff64(unsigned long long x, unsigned long long y)
+static inline unsigned long long absdiff64(
+	unsigned long long x,
+	unsigned long long y)
 {
 	return x < y ? y - x : x - y;
 }
@@ -53,6 +60,8 @@ extern uint64_t host_to_little_u64(uint64_t v);
 extern uint64_t big_to_host_u64(uint64_t v);
 extern uint64_t little_to_host_u64(uint64_t v);
 
+extern bool fill_random(void *buffer, unsigned int size);
+
 #ifdef __cplusplus
 }
 #endif

include/mango/vm-object.h

@@ -0,0 +1,68 @@
+#ifndef MANGO_VM_OBJECT_H_
+#define MANGO_VM_OBJECT_H_
+
+#include <mango/locks.h>
+#include <mango/object.h>
+
+#define VM_OBJECT_NAME_MAX 64
+
+enum vm_object_flags {
+	/* the memory behind this vm-object wasn't allocated by us, and
+	 * therefore shouldn't be freed by us */
+	VMO_IN_PLACE = 0x01u,
+};
+
+struct vm_object {
+	struct object vo_base;
+
+	char vo_name[VM_OBJECT_NAME_MAX];
+	enum vm_object_flags vo_flags;
+
+	/* queue of struct vm_region_mapping */
+	struct queue vo_mappings;
+
+	/* memory protection flags. mappings of this vm_object can only use
+	 * a subset of the flags set in this mask. */
+	enum vm_prot vo_prot;
+
+	/* btree of vm_pages that have been allocated to this vm_object.
+	 * vm_page->p_vmo_offset and the size of each page is the bst key. */
+	struct btree vo_pages;
+	/* total length of the vm_object in bytes. */
+	size_t vo_size;
+};
+
+extern kern_status_t vm_object_type_init(void);
+
+/* create a vm_object with the specified length in bytes and protection flags.
+ * the length will be automatically rounded up to the nearest vm_object page
+ * order size. the actual page frames themselves won't be allocated until
+ * they are mapped and accessed. */
+extern struct vm_object *vm_object_create(
+	const char *name,
+	size_t len,
+	enum vm_prot prot);
+
+/* create a vm_object that represents the specified range of physical memory.
+ * the length will be automatically rounded up to the nearest vm_object page
+ * order size.
+ * NOTE this function assumes that the physical memory has already been
+ * reserved, and is not in use by any other kernel component. */
+extern struct vm_object *vm_object_create_in_place(
+	const char *name,
+	phys_addr_t base,
+	size_t len,
+	enum vm_prot prot);
+
+extern struct vm_page *vm_object_get_page(
+	const struct vm_object *vo,
+	off_t offset);
+
+extern struct vm_page *vm_object_alloc_page(
+	struct vm_object *vo,
+	off_t offset,
+	enum vm_page_order size);
+
+DEFINE_OBJECT_LOCK_FUNCTION(vm_object, vo_base)
+
+#endif

include/mango/vm-region.h

@@ -0,0 +1,126 @@
+#ifndef MANGO_VM_REGION_H_
+#define MANGO_VM_REGION_H_
+
+#include <mango/object.h>
+#include <mango/pmap.h>
+#include <mango/vm.h>
+
+#define VM_REGION_NAME_MAX        64
+
+#define VM_REGION_ANY_MAP_ADDRESS ((virt_addr_t) - 1)
+
+struct vm_region;
+struct vm_object;
+
+enum vm_region_entry_type {
+	VM_REGION_ENTRY_NONE = 0,
+	VM_REGION_ENTRY_REGION,
+	VM_REGION_ENTRY_MAPPING,
+};
+
+struct vm_region_entry {
+	struct btree_node e_node;
+	struct vm_region_entry *e_parent;
+	enum vm_region_entry_type e_type;
+	/* absolute virtual address of the entry */
+	virt_addr_t e_base_address;
+	/* size of the entry in bytes */
+	size_t e_size;
+};
+
+struct vm_region_mapping {
+	struct vm_region_entry m_entry;
+	struct vm_object *m_object;
+
+	/* used to link to  vm_object->vo_mappings */
+	struct queue_entry m_object_entry;
+
+	enum vm_prot m_prot;
+	/* offset in bytes to the start of the object data that was mapped */
+	off_t m_object_offset;
+};
+
+struct vm_region {
+	struct object vr_base;
+	struct vm_region_entry vr_entry;
+
+	char vr_name[VM_REGION_NAME_MAX];
+
+	/* btree of struct vm_region_entry.
+	 * sibling entries cannot overlap each other, and child entries must
+	 * be entirely contained within their immediate parent entry. */
+	struct btree vr_entries;
+
+	/* memory protection restriction mask.
+	 * any mapping in this region, or any of its children, cannot use
+	 * protection flags that are not set in this mask.
+	 * for example, if VM_PROT_EXEC is /not/ set here, no mapping
+	 * can be created in this region or any child region with VM_PROT_EXEC
+	 * set. */
+	enum vm_prot vr_prot;
+
+	/* the physical address space in which mappings in this region (and
+	 * its children) are created */
+	pmap_t vr_pmap;
+};
+
+extern kern_status_t vm_region_type_init(void);
+
+extern kern_status_t vm_region_create(
+	struct vm_region *parent,
+	const char *name,
+	virt_addr_t base,
+	size_t len,
+	enum vm_prot prot,
+	struct vm_region **out);
+
+/* find the child region that has jurisdiction over the specified virtual
+ * address. returns the lowest-nested region that covers the specified virtual
+ * address. */
+extern struct vm_region *vm_region_find_child(
+	struct vm_region *region,
+	virt_addr_t addr);
+
+/* find the child region that has jurisdiction over the specified virtual
+ * address area. returns the lowest-nested region that covers the specified
+ * virtual address area. the area must be fully contained within a region, with
+ * no partial overlaps. if an area is covered by multiple regions, or is only
+ * partially within a region, returns NULL. */
+extern struct vm_region *vm_region_find_child_for_area(
+	struct vm_region *region,
+	virt_addr_t addr,
+	size_t len);
+extern struct vm_region_mapping *vm_region_find_mapping(
+	struct vm_region *region,
+	virt_addr_t addr);
+
+extern kern_status_t vm_region_map_object(
+	struct vm_region *region,
+	virt_addr_t map_address,
+	struct vm_object *object,
+	off_t object_offset,
+	size_t length,
+	enum vm_prot prot,
+	virt_addr_t *out);
+
+/* returns true if the memory area defined by [base, base+len] contains:
+ *  - no child regions
+ *  - no vm_object mappings
+ * if any child regions or mappings exist in the memory area, returns false.
+ * if the memory area exceeds the bounds of the region, returns false.
+ */
+extern bool vm_region_is_area_free(
+	const struct vm_region *region,
+	virt_addr_t base,
+	size_t len);
+
+extern kern_status_t vm_region_demand_map(
+	struct vm_region *region,
+	virt_addr_t addr,
+	enum pmap_fault_flags flags);
+
+extern void vm_region_dump(struct vm_region *region, int depth);
+
+DEFINE_OBJECT_LOCK_FUNCTION(vm_region, vr_base)
+
+#endif

include/mango/vm.h

@@ -1,14 +1,14 @@
 #ifndef MANGO_VM_H_
 #define MANGO_VM_H_
 
-#include <stddef.h>
-#include <mango/types.h>
-#include <mango/status.h>
-#include <mango/queue.h>
-#include <mango/btree.h>
 #include <mango/bitmap.h>
+#include <mango/btree.h>
 #include <mango/locks.h>
 #include <mango/machine/vm.h>
+#include <mango/queue.h>
+#include <mango/status.h>
+#include <mango/types.h>
+#include <stddef.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -17,13 +17,13 @@ extern "C" {
 struct bcache;
 
 /* maximum number of NUMA nodes */
-#define VM_MAX_NODES 64
+#define VM_MAX_NODES            64
 /* maximum number of memory zones per node */
-#define VM_MAX_ZONES (VM_ZONE_MAX + 1)
+#define VM_MAX_ZONES            (VM_ZONE_MAX + 1)
 /* maximum number of supported page orders */
-#define VM_MAX_PAGE_ORDERS (VM_PAGE_MAX_ORDER + 1)
+#define VM_MAX_PAGE_ORDERS      (VM_PAGE_MAX_ORDER + 1)
 /* maximum number of sparse memory sectors */
-#define VM_MAX_SECTORS 8192
+#define VM_MAX_SECTORS          8192
 
 /* maximum number of disk sectors that can be stored in a single
    page. AKA the number of bits in the sector bitmap.
@@ -33,44 +33,41 @@ struct bcache;
 #define VM_CHECK_ALIGN(p, mask) ((((p) & (mask)) == (p)) ? 1 : 0)
 
 #define VM_CACHE_INITIALISED(c) ((c)->c_obj_count != 0)
-#define VM_PAGE_IS_FREE(pg) (((pg)->p_flags & (VM_PAGE_RESERVED | VM_PAGE_ALLOC)) == 0)
+#define VM_PAGE_IS_FREE(pg)                                                    \
+	(((pg)->p_flags & (VM_PAGE_RESERVED | VM_PAGE_ALLOC)) == 0)
 
-#define vm_page_foreach(pg, i) \
+#define vm_page_foreach(pg, i)                                                 \
 	for (struct vm_page *i = (pg); i; i = vm_page_get_next_tail(i))
 
 typedef phys_addr_t vm_alignment_t;
 typedef unsigned int vm_node_id_t;
 
-struct vm_object {
-	unsigned int reserved;
-};
-
 enum vm_model {
 	VM_MODEL_FLAT = 1,
 	VM_MODEL_SPARSE,
 };
 
 enum vm_prot {
-	VM_PROT_READ    = 0x01u,
-	VM_PROT_WRITE   = 0x02u,
-	VM_PROT_EXEC    = 0x04u,
-	VM_PROT_USER    = 0x08u,
-	VM_PROT_SVR     = 0x10u,
+	VM_PROT_READ = 0x01u,
+	VM_PROT_WRITE = 0x02u,
+	VM_PROT_EXEC = 0x04u,
+	VM_PROT_USER = 0x08u,
+	VM_PROT_SVR = 0x10u,
 	VM_PROT_NOCACHE = 0x20u,
 };
 
 enum vm_flags {
-	VM_NORMAL  = 0x00u,
+	VM_NORMAL = 0x00u,
 	VM_GET_DMA = 0x01u,
 };
 
 enum vm_zone_id {
-	/* NOTE that these are used as indices into the node_zones array in vm/zone.c
-	   they need to be continuous, and must start at 0!
+	/* NOTE that these are used as indices into the node_zones array in
+	   vm/zone.c they need to be continuous, and must start at 0!
 
 	   not all of these zones are implemented for every architecture. */
-	VM_ZONE_DMA     = 0u,
-	VM_ZONE_NORMAL  = 1u,
+	VM_ZONE_DMA = 0u,
+	VM_ZONE_NORMAL = 1u,
 	VM_ZONE_HIGHMEM = 2u,
 };
 
@@ -108,27 +105,28 @@ enum vm_page_order {
 };
 
 enum vm_page_flags {
-	/* page is reserved (probably by a call to memblock_reserve()) and cannot be
-	   returned by any allocation function */
-	VM_PAGE_RESERVED   = 0x01u,
+	/* page is reserved (probably by a call to memblock_reserve()) and
+	   cannot be returned by any allocation function */
+	VM_PAGE_RESERVED = 0x01u,
 	/* page has been allocated by a zone's buddy allocator, and is in-use */
-	VM_PAGE_ALLOC      = 0x02u,
+	VM_PAGE_ALLOC = 0x02u,
 	/* page is the first page of a huge-page */
-	VM_PAGE_HEAD       = 0x04u,
+	VM_PAGE_HEAD = 0x04u,
 	/* page is part of a huge-page */
-	VM_PAGE_HUGE       = 0x08u,
-	/* page is holding cached data from secondary storage, and can be freed if necessary (and not dirty). */
-	VM_PAGE_CACHE      = 0x10u,
+	VM_PAGE_HUGE = 0x08u,
+	/* page is holding cached data from secondary storage, and can be freed
+	 * if necessary (and not dirty). */
+	VM_PAGE_CACHE = 0x10u,
 };
 
 enum vm_memory_region_status {
-	VM_REGION_FREE		= 0x01u,
-	VM_REGION_RESERVED	= 0x02u,
+	VM_REGION_FREE = 0x01u,
+	VM_REGION_RESERVED = 0x02u,
 };
 
 enum vm_cache_flags {
 	VM_CACHE_OFFSLAB = 0x01u,
-	VM_CACHE_DMA     = 0x02u
+	VM_CACHE_DMA = 0x02u
 };
 
 struct vm_zone_descriptor {
@@ -151,12 +149,6 @@ struct vm_pg_data {
 	struct vm_zone pg_zones[VM_MAX_ZONES];
 };
 
-struct vm_region {
-	enum vm_memory_region_status r_status;
-	phys_addr_t r_base;
-	phys_addr_t r_limit;
-};
-
 struct vm_cache {
 	const char *c_name;
 	enum vm_cache_flags c_flags;
@@ -204,7 +196,7 @@ struct vm_slab {
 	     - s_freelist[s_free] should be set to the previous value of s_free.
 	   this is commented as it as flexible arrays are not supported in c++.
 	   */
-	//unsigned int s_freelist[];
+	// unsigned int s_freelist[];
 };
 
 struct vm_page {
@@ -231,20 +223,25 @@ struct vm_page {
 	/* multi-purpose list/tree entry.
 	   the owner of the page can decide what to do with this.
 	   some examples:
-	     - the buddy allocator uses this to maintain its per-zone free-page lists.
-	     - the block cache uses this to maintain a tree of pages keyed by block number.
-        */
+	     - the buddy allocator uses this to maintain its per-zone free-page
+	       lists.
+	     - vm_object uses it to maintain a btree of allocated pages keyed
+	       by offset/size.
+	     - the block cache uses this to maintain a tree of pages keyed by
+	       block number.
+	*/
 	union {
 		struct queue_entry p_list;
 		struct btree_node p_bnode;
 
-		/* btree_node contains three pointers, so provide three pointer-sized integers for
-		   use if p_bnode isn't needed. */
+		/* btree_node contains three pointers, so provide three
+		   pointer-sized integers for use if p_bnode isn't needed. */
 		uintptr_t priv1[3];
 	};
 
 	union {
-		/* used by bcache when sector size is < page size. bitmap of present/missing sectors */
+		/* used by bcache when sector size is < page size. bitmap of
+		 * present/missing sectors */
 		DECLARE_BITMAP(p_blockbits, VM_MAX_SECTORS_PER_PAGE);
 		uint32_t p_priv2;
 	};
@@ -252,10 +249,12 @@ struct vm_page {
 	union {
 		/* sector address, used by bcache */
 		sectors_t p_blockid;
+		/* offset of this page within the vm_object it is a part of */
+		off_t p_vmo_offset;
 
 		uint32_t p_priv3[2];
 	};
-} __attribute__((aligned(2 * sizeof(unsigned long))));
+} __aligned(2 * sizeof(unsigned long));
 
 /* represents a sector of memory, containing its own array of vm_pages.
    this struct is used under the sparse memory model, instead of the
@@ -272,39 +271,58 @@ struct vm_sector {
 	struct vm_page *s_pages;
 };
 
-extern kern_status_t vm_bootstrap(const struct vm_zone_descriptor *zones, size_t nr_zones);
+extern kern_status_t vm_bootstrap(
+	const struct vm_zone_descriptor *zones,
+	size_t nr_zones);
 extern enum vm_model vm_memory_model(void);
 extern void vm_set_memory_model(enum vm_model model);
 
 extern struct vm_pg_data *vm_pg_data_get(vm_node_id_t node);
 
-extern phys_addr_t vm_virt_to_phys(void *p);
+extern phys_addr_t vm_virt_to_phys(const void *p);
 extern void *vm_phys_to_virt(phys_addr_t p);
 
-extern void vm_page_init_array();
+extern size_t vm_page_order_to_bytes(enum vm_page_order order);
+extern size_t vm_page_order_to_pages(enum vm_page_order order);
+extern vm_alignment_t vm_page_order_to_alignment(enum vm_page_order order);
+extern void vm_page_init_array(void);
 extern struct vm_page *vm_page_get(phys_addr_t addr);
 extern phys_addr_t vm_page_get_paddr(struct vm_page *pg);
 extern struct vm_zone *vm_page_get_zone(struct vm_page *pg);
 extern void *vm_page_get_vaddr(struct vm_page *pg);
 extern size_t vm_page_get_pfn(struct vm_page *pg);
-extern size_t vm_page_order_to_bytes(enum vm_page_order order);
-extern size_t vm_page_order_to_pages(enum vm_page_order order);
-extern vm_alignment_t vm_page_order_to_alignment(enum vm_page_order order);
-extern struct vm_page *vm_page_alloc(enum vm_page_order order, enum vm_flags flags);
+static inline size_t vm_page_get_size_bytes(const struct vm_page *pg)
+{
+	return vm_page_order_to_bytes(pg->p_order);
+}
+extern struct vm_page *vm_page_alloc(
+	enum vm_page_order order,
+	enum vm_flags flags);
 extern void vm_page_free(struct vm_page *pg);
 
-extern int vm_page_split(struct vm_page *pg, struct vm_page **a, struct vm_page **b);
+extern int vm_page_split(
+	struct vm_page *pg,
+	struct vm_page **a,
+	struct vm_page **b);
 extern struct vm_page *vm_page_merge(struct vm_page *a, struct vm_page *b);
 extern struct vm_page *vm_page_get_buddy(struct vm_page *pg);
 extern struct vm_page *vm_page_get_next_tail(struct vm_page *pg);
 
 extern size_t vm_bytes_to_pages(size_t bytes);
 
-extern void vm_zone_init(struct vm_zone *z, const struct vm_zone_descriptor *zone_info);
-extern struct vm_page *vm_zone_alloc_page(struct vm_zone *z, enum vm_page_order order, enum vm_flags flags);
+extern void vm_zone_init(
+	struct vm_zone *z,
+	const struct vm_zone_descriptor *zone_info);
+extern struct vm_page *vm_zone_alloc_page(
+	struct vm_zone *z,
+	enum vm_page_order order,
+	enum vm_flags flags);
 extern void vm_zone_free_page(struct vm_zone *z, struct vm_page *pg);
 
-extern struct vm_cache *vm_cache_create(const char *name, size_t objsz, enum vm_cache_flags flags);
+extern struct vm_cache *vm_cache_create(
+	const char *name,
+	size_t objsz,
+	enum vm_cache_flags flags);
 extern void vm_cache_init(struct vm_cache *cache);
 extern void vm_cache_destroy(struct vm_cache *cache);
 extern void *vm_cache_alloc(struct vm_cache *cache, enum vm_flags flags);
@@ -330,15 +348,18 @@ extern size_t vm_page_get_pfn_sparse(struct vm_page *pg);
 #endif
 
 #ifdef __cplusplus
-inline void *operator new(size_t count, void *p) { return p; }
+inline void *operator new(size_t count, void *p)
+{
+	return p;
+}
 
-#define kmalloc_object(objtype, flags, ...)        \
-        __extension__({                            \
-	void *p = kmalloc(sizeof(objtype), flags); \
-	if (p) {                                   \
-		new (p) objtype(__VA_ARGS__);      \
-	}                                          \
-	(objtype *)p;                              \
+#define kmalloc_object(objtype, flags, ...)                                    \
+	__extension__({                                                        \
+		void *p = kmalloc(sizeof(objtype), flags);                     \
+		if (p) {                                                       \
+			new (p) objtype(__VA_ARGS__);                          \
+		}                                                              \
+		(objtype *)p;                                                  \
 	})
 
 #endif

init/main.c

@@ -1,6 +1,8 @@
 #include <mango/arg.h>
+#include <mango/bsp.h>
 #include <mango/clock.h>
 #include <mango/cpu.h>
+#include <mango/handle.h>
 #include <mango/init.h>
 #include <mango/input.h>
 #include <mango/libc/stdio.h>
@@ -10,6 +12,7 @@
 #include <mango/printk.h>
 #include <mango/sched.h>
 #include <mango/test.h>
+#include <mango/vm-object.h>
 #include <stdint.h>
 
 extern unsigned long get_rflags(void);
@@ -23,14 +26,16 @@ void print_kernel_banner(void)
 
 static void hang(void)
 {
-	struct task *self = current_task();
-	struct thread *thread = current_thread();
+	// struct task *self = current_task();
+	// struct thread *thread = current_thread();
 
 	while (1) {
+#if 0
 		printk("[cpu %u, task %u, thread %u]: tick",
 		       this_cpu(),
 		       self->t_id,
 		       thread->tr_id);
+#endif
 		milli_sleep(2000);
 	}
 }
@@ -67,9 +72,40 @@ void kernel_init(uintptr_t arg)
 {
 	ml_init(arg);
 
-	printk("kernel_init() running on processor %u", this_cpu());
+	struct boot_module bsp_image = {0};
+	bsp_get_location(&bsp_image);
 
-	create_kernel_thread(background_thread);
+	tracek("kernel_init() running on processor %u", this_cpu());
+
+	if (!bsp_image.mod_base) {
+		printk("FATAL: no bsp image specified");
+		hang();
+	}
+
+	tracek("bsp image at [0x%llx-0x%llx]",
+	       bsp_image.mod_base,
+	       bsp_image.mod_base + bsp_image.mod_size);
+
+	struct bsp bsp;
+	kern_status_t status = bsp_load(&bsp, &bsp_image);
+	if (status != KERN_OK) {
+		printk("FATAL: bsp image is corrupt/invalid");
+		hang();
+	}
+
+	tracek("bsp image loaded. text=[%06llx-%06llx], data=[%06llx-%06llx], "
+	       "entry=%06llx, vmo=%p",
+	       bsp.bsp_trailer.bsp_text_vaddr,
+	       bsp.bsp_trailer.bsp_text_size + bsp.bsp_trailer.bsp_text_vaddr,
+	       bsp.bsp_trailer.bsp_data_vaddr,
+	       bsp.bsp_trailer.bsp_data_size + bsp.bsp_trailer.bsp_data_vaddr,
+	       bsp.bsp_trailer.bsp_exec_entry,
+	       bsp.bsp_vmo);
+
+	struct task *bootstrap_task = task_create(kernel_task(), "bootstrap");
+	tracek("created bootstrap task (pid=%u)", bootstrap_task->t_id);
+
+	bsp_launch_async(&bsp, bootstrap_task);
 
 	hang();
 }

kernel/bsp.c

@@ -0,0 +1,190 @@
+#include <mango/bsp.h>
+#include <mango/printk.h>
+#include <mango/sched.h>
+#include <mango/util.h>
+#include <mango/vm-object.h>
+#include <mango/vm-region.h>
+
+#define BOOTSTRAP_STACK_SIZE 0x10000
+
+static struct boot_module bsp_location = {0};
+
+void bsp_set_location(const struct boot_module *mod)
+{
+	memcpy(&bsp_location, mod, sizeof bsp_location);
+}
+
+void bsp_get_location(struct boot_module *out)
+{
+	memcpy(out, &bsp_location, sizeof bsp_location);
+}
+
+kern_status_t bsp_load(struct bsp *bsp, const struct boot_module *mod)
+{
+	size_t trailer_offset = mod->mod_size - sizeof(struct bsp_trailer);
+	const char *p = vm_phys_to_virt(mod->mod_base);
+	const struct bsp_trailer *trailer_bigendian
+		= (const struct bsp_trailer *)(p + trailer_offset);
+
+	bsp->bsp_trailer.bsp_magic
+		= big_to_host_u32(trailer_bigendian->bsp_magic);
+	bsp->bsp_trailer.bsp_fs_offset
+		= big_to_host_u64(trailer_bigendian->bsp_fs_offset);
+	bsp->bsp_trailer.bsp_fs_len
+		= big_to_host_u32(trailer_bigendian->bsp_fs_len);
+	bsp->bsp_trailer.bsp_exec_offset
+		= big_to_host_u64(trailer_bigendian->bsp_exec_offset);
+	bsp->bsp_trailer.bsp_exec_len
+		= big_to_host_u32(trailer_bigendian->bsp_exec_len);
+	bsp->bsp_trailer.bsp_text_faddr
+		= big_to_host_u64(trailer_bigendian->bsp_text_faddr);
+	bsp->bsp_trailer.bsp_text_vaddr
+		= big_to_host_u64(trailer_bigendian->bsp_text_vaddr);
+	bsp->bsp_trailer.bsp_text_size
+		= big_to_host_u64(trailer_bigendian->bsp_text_size);
+	bsp->bsp_trailer.bsp_data_faddr
+		= big_to_host_u64(trailer_bigendian->bsp_data_faddr);
+	bsp->bsp_trailer.bsp_data_vaddr
+		= big_to_host_u64(trailer_bigendian->bsp_data_vaddr);
+	bsp->bsp_trailer.bsp_data_size
+		= big_to_host_u64(trailer_bigendian->bsp_data_size);
+	bsp->bsp_trailer.bsp_exec_entry
+		= big_to_host_u64(trailer_bigendian->bsp_exec_entry);
+
+	if (bsp->bsp_trailer.bsp_magic != BSP_MAGIC) {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	bsp->bsp_vmo = vm_object_create_in_place(
+		"bsp",
+		mod->mod_base,
+		mod->mod_size,
+		VM_PROT_READ | VM_PROT_EXEC | VM_PROT_USER);
+	if (!bsp->bsp_vmo) {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	return KERN_OK;
+}
+
+static kern_status_t map_executable(
+	struct bsp *bsp,
+	struct task *task,
+	virt_addr_t *entry)
+{
+	kern_status_t status = KERN_OK;
+	size_t exec_size = 0;
+	if (bsp->bsp_trailer.bsp_text_vaddr > bsp->bsp_trailer.bsp_data_vaddr) {
+		exec_size = bsp->bsp_trailer.bsp_text_vaddr
+		          + bsp->bsp_trailer.bsp_text_size;
+	} else {
+		exec_size = bsp->bsp_trailer.bsp_data_vaddr
+		          + bsp->bsp_trailer.bsp_data_size;
+	}
+
+	struct vm_region *region;
+	status = vm_region_create(
+		task->t_address_space,
+		"exec",
+		VM_REGION_ANY_MAP_ADDRESS,
+		exec_size,
+		VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXEC | VM_PROT_USER,
+		&region);
+	if (status != KERN_OK) {
+		return status;
+	}
+
+	struct vm_object *data = vm_object_create(
+		".data",
+		bsp->bsp_trailer.bsp_data_size,
+		VM_PROT_READ | VM_PROT_WRITE | VM_PROT_USER);
+	/* TODO copy .data from executable to memory */
+	if (!data) {
+		return KERN_NO_MEMORY;
+	}
+
+	off_t text_offset = bsp->bsp_trailer.bsp_exec_offset
+	                  + bsp->bsp_trailer.bsp_text_faddr;
+	off_t data_offset = 0;
+	virt_addr_t text_base = region->vr_entry.e_base_address
+	                      + bsp->bsp_trailer.bsp_text_vaddr;
+	virt_addr_t data_base = region->vr_entry.e_base_address
+	                      + bsp->bsp_trailer.bsp_data_vaddr;
+
+	tracek("exec_offset=%llx, text_faddr=%llx",
+	       bsp->bsp_trailer.bsp_exec_offset,
+	       bsp->bsp_trailer.bsp_text_faddr);
+	tracek("text_offset=%llx, data_offset=%llx", text_offset, data_offset);
+	tracek("text_base=%llx, data_base=%llx", text_base, data_base);
+
+	status = vm_region_map_object(
+		region,
+		text_base,
+		bsp->bsp_vmo,
+		text_offset,
+		bsp->bsp_trailer.bsp_text_size,
+		VM_PROT_READ | VM_PROT_EXEC | VM_PROT_USER,
+		&text_base);
+	if (status != KERN_OK) {
+		return status;
+	}
+
+	status = vm_region_map_object(
+		region,
+		data_base,
+		data,
+		data_offset,
+		bsp->bsp_trailer.bsp_data_size,
+		VM_PROT_READ | VM_PROT_WRITE | VM_PROT_USER,
+		&data_base);
+	if (status != KERN_OK) {
+		return status;
+	}
+
+	*entry = text_base + bsp->bsp_trailer.bsp_exec_entry;
+	return KERN_OK;
+}
+
+kern_status_t bsp_launch_async(struct bsp *bsp, struct task *task)
+{
+	virt_addr_t stack_buffer;
+	virt_addr_t entry, sp;
+	kern_status_t status = map_executable(bsp, task, &entry);
+	if (status != KERN_OK) {
+		return status;
+	}
+
+	struct vm_object *user_stack = vm_object_create(
+		"stack",
+		BOOTSTRAP_STACK_SIZE,
+		VM_PROT_READ | VM_PROT_WRITE | VM_PROT_USER);
+	if (!user_stack) {
+		return KERN_NO_ENTRY;
+	}
+
+	status = vm_region_map_object(
+		task->t_address_space,
+		VM_REGION_ANY_MAP_ADDRESS,
+		user_stack,
+		0,
+		BOOTSTRAP_STACK_SIZE,
+		VM_PROT_READ | VM_PROT_WRITE | VM_PROT_USER,
+		&stack_buffer);
+
+	if (status != KERN_OK) {
+		return status;
+	}
+
+#ifdef TRACE
+	vm_region_dump(task->t_address_space, 0);
+#endif
+
+	sp = stack_buffer + BOOTSTRAP_STACK_SIZE;
+	tracek("bootstrap: entry=%llx, sp=%llx", entry, sp);
+
+	struct thread *init_thread = task_create_thread(task);
+	thread_init_user(init_thread, entry, sp);
+	schedule_thread_on_cpu(init_thread);
+
+	return KERN_OK;
+}

kernel/handle.c

@@ -0,0 +1,177 @@
+#include <mango/handle.h>
+#include <mango/libc/string.h>
+#include <mango/object.h>
+#include <mango/vm.h>
+
+/* depth=3 gives a maximum of ~66.6 million handles */
+#define MAX_TABLE_DEPTH 3
+
+static struct vm_cache handle_table_cache = {
+	.c_name = "handle_table",
+	.c_obj_size = sizeof(struct handle_table),
+};
+
+struct handle_table *handle_table_create(void)
+{
+	if (!VM_CACHE_INITIALISED(&handle_table_cache)) {
+		vm_cache_init(&handle_table_cache);
+	}
+
+	struct handle_table *out
+		= vm_cache_alloc(&handle_table_cache, VM_NORMAL);
+	if (!out) {
+		return NULL;
+	}
+
+	memset(out, 0x0, sizeof *out);
+
+	return out;
+}
+
+void handle_table_destroy(struct handle_table *tab)
+{
+}
+
+static kern_status_t decode_handle_indices(
+	kern_handle_t handle,
+	unsigned int indices[MAX_TABLE_DEPTH])
+{
+	handle >>= 2;
+	for (int i = 0; i < MAX_TABLE_DEPTH; i++) {
+		unsigned int div = (i > 0 ? REFS_PER_TABLE : HANDLES_PER_TABLE);
+
+		unsigned int v = handle % div;
+		indices[MAX_TABLE_DEPTH - i - 1] = v;
+		handle /= div;
+	}
+
+	return handle == 0 ? KERN_OK : KERN_INVALID_ARGUMENT;
+}
+
+static kern_status_t encode_handle_indices(
+	unsigned int indices[MAX_TABLE_DEPTH],
+	kern_handle_t *out_handle)
+{
+	kern_handle_t handle = 0;
+	unsigned int mul = 1;
+
+	for (int i = MAX_TABLE_DEPTH - 1; i >= 0; i--) {
+		unsigned int v = indices[i] * mul;
+		handle += v;
+		mul *= REFS_PER_TABLE;
+	}
+
+	handle <<= 2;
+
+	*out_handle = handle;
+	return KERN_OK;
+}
+
+kern_status_t handle_table_alloc_handle(
+	struct handle_table *tab,
+	struct handle **out_slot,
+	kern_handle_t *out_handle)
+{
+	int i;
+	unsigned int indices[MAX_TABLE_DEPTH] = {0};
+
+	for (i = 0; i < MAX_TABLE_DEPTH - 1; i++) {
+		unsigned int next_index = bitmap_lowest_clear(
+			tab->t_subtables.t_subtable_map,
+			REFS_PER_TABLE);
+
+		if (next_index == BITMAP_NPOS) {
+			return KERN_NO_ENTRY;
+		}
+
+		struct handle_table *next
+			= tab->t_subtables.t_subtable_list[next_index];
+		if (!next) {
+			next = handle_table_create();
+			tab->t_subtables.t_subtable_list[next_index] = next;
+		}
+
+		if (!next) {
+			return KERN_NO_MEMORY;
+		}
+
+		indices[i] = next_index;
+		tab = next;
+	}
+
+	unsigned int handle_index = bitmap_lowest_clear(
+		tab->t_handles.t_handle_map,
+		HANDLES_PER_TABLE);
+	if (handle_index == BITMAP_NPOS) {
+		return KERN_NO_ENTRY;
+	}
+
+	bitmap_set(tab->t_handles.t_handle_map, handle_index);
+	memset(&tab->t_handles.t_handle_list[handle_index],
+	       0x0,
+	       sizeof(struct handle));
+
+	indices[i] = handle_index;
+
+	*out_slot = &tab->t_handles.t_handle_list[handle_index];
+	return encode_handle_indices(indices, out_handle);
+}
+
+void handle_table_free_handle(struct handle_table *tab, kern_handle_t handle)
+{
+	unsigned int indices[MAX_TABLE_DEPTH];
+	if (decode_handle_indices(handle, indices) != KERN_OK) {
+		return;
+	}
+
+	int i;
+	for (i = 0; i < MAX_TABLE_DEPTH - 1; i++) {
+		struct handle_table *next
+			= tab->t_subtables.t_subtable_list[indices[i]];
+		if (!next) {
+			return;
+		}
+
+		bitmap_clear(tab->t_subtables.t_subtable_map, indices[i]);
+		tab = next;
+	}
+
+	unsigned int handle_index = indices[i];
+	bitmap_clear(tab->t_handles.t_handle_map, handle_index);
+	struct handle *handle_entry
+		= &tab->t_handles.t_handle_list[handle_index];
+
+	if (handle_entry->h_object) {
+		object_remove_handle(handle_entry->h_object);
+	}
+
+	memset(handle_entry, 0x0, sizeof *handle_entry);
+}
+
+struct handle *handle_table_get_handle(
+	struct handle_table *tab,
+	kern_handle_t handle)
+{
+	unsigned int indices[MAX_TABLE_DEPTH];
+	if (decode_handle_indices(handle, indices) != KERN_OK) {
+		return NULL;
+	}
+
+	int i;
+	for (i = 0; i < MAX_TABLE_DEPTH - 1; i++) {
+		struct handle_table *next
+			= tab->t_subtables.t_subtable_list[indices[i]];
+		if (!next) {
+			return NULL;
+		}
+
+		tab = next;
+	}
+
+	unsigned int handle_index = indices[i];
+	if (!bitmap_check(tab->t_handles.t_handle_map, handle_index)) {
+		return NULL;
+	}
+
+	return &tab->t_handles.t_handle_list[handle_index];
+}

kernel/object.c

@@ -21,7 +21,6 @@ kern_status_t object_type_register(struct object_type *p)
 
 	p->ob_cache.c_name = p->ob_name;
 	p->ob_cache.c_obj_size = p->ob_size;
-	p->ob_cache.c_page_order = VM_PAGE_16K;
 
 	vm_cache_init(&p->ob_cache);
 	p->ob_flags |= OBJTYPE_INIT;
@@ -71,7 +70,21 @@ struct object *object_ref(struct object *obj)
 	return obj;
 }
 
-void object_deref(struct object *obj)
+static void object_cleanup(struct object *obj, unsigned long flags)
+{
+	if (obj->ob_refcount > 0 || obj->ob_handles > 0) {
+		spin_unlock_irqrestore(&obj->ob_lock, flags);
+		return;
+	}
+
+	if (HAS_OP(obj, destroy)) {
+		obj->ob_type->ob_ops.destroy(obj);
+	}
+
+	vm_cache_free(&obj->ob_type->ob_cache, obj);
+}
+
+void object_unref(struct object *obj)
 {
 	unsigned long flags;
 	spin_lock_irqsave(&obj->ob_lock, &flags);
@@ -82,17 +95,26 @@ void object_deref(struct object *obj)
 	}
 
 	obj->ob_refcount--;
+	object_cleanup(obj, flags);
+}
 
-	if (obj->ob_refcount > 0) {
+void object_add_handle(struct object *obj)
+{
+	obj->ob_handles++;
+}
+
+void object_remove_handle(struct object *obj)
+{
+	unsigned long flags;
+	spin_lock_irqsave(&obj->ob_lock, &flags);
+
+	if (obj->ob_handles == 0) {
 		spin_unlock_irqrestore(&obj->ob_lock, flags);
 		return;
 	}
 
-	if (HAS_OP(obj, destroy)) {
-		obj->ob_type->ob_ops.destroy(obj);
-	}
-
-	vm_cache_free(&obj->ob_type->ob_cache, obj);
+	obj->ob_handles--;
+	object_cleanup(obj, flags);
 }
 
 void object_lock(struct object *obj)

kernel/panic.c

@@ -1,13 +1,13 @@
-#include <stdarg.h>
-#include <mango/machine/panic.h>
+#include <mango/cpu.h>
 #include <mango/libc/stdio.h>
+#include <mango/machine/panic.h>
 #include <mango/printk.h>
 #include <mango/sched.h>
-#include <mango/cpu.h>
+#include <stdarg.h>
 
 static int has_panicked = 0;
 
-void panic_irq(struct cpu_context *ctx, const char *fmt, ...)
+void panic_irq(struct ml_cpu_context *ctx, const char *fmt, ...)
 {
 	char buf[512];
 	va_list args;
@@ -22,7 +22,10 @@ void panic_irq(struct cpu_context *ctx, const char *fmt, ...)
 	struct thread *thr = current_thread();
 
 	if (task && thr) {
-		printk("task: %s (id: %d, thread: %d)", task->t_name, task->t_id, thr->tr_id);
+		printk("task: %s (id: %d, thread: %d)",
+		       task->t_name,
+		       task->t_id,
+		       thr->tr_id);
 	} else {
 		printk("task: [bootstrap]");
 	}

kernel/syscall.c

@@ -0,0 +1,40 @@
+#include <mango/machine/cpu.h>
+#include <mango/printk.h>
+#include <mango/syscall.h>
+
+kern_status_t sys_exit(int status)
+{
+	printk("sys_exit(%d)", status);
+	while (1) {
+		ml_cpu_pause();
+	}
+	return KERN_UNIMPLEMENTED;
+}
+
+kern_status_t sys_vm_object_create(
+	const char *name,
+	size_t len,
+	enum vm_prot prot,
+	kern_handle_t *out_handle)
+{
+	printk("sys_vm_object_create()");
+	return KERN_UNIMPLEMENTED;
+}
+
+#define SYSCALL_TABLE_ENTRY(id, p) [SYS_##id] = (virt_addr_t)(sys_##p)
+
+static const virt_addr_t syscall_table[] = {
+	SYSCALL_TABLE_ENTRY(EXIT, exit),
+	SYSCALL_TABLE_ENTRY(VM_OBJECT_CREATE, vm_object_create),
+};
+static const size_t syscall_table_count
+	= sizeof syscall_table / sizeof syscall_table[0];
+
+virt_addr_t syscall_get_func(unsigned int sysid)
+{
+	if (sysid >= syscall_table_count) {
+		return 0;
+	}
+
+	return syscall_table[sysid];
+}

libc/ctype/ctype.c

@@ -18,77 +18,76 @@
 #include <stdbool.h>
 #include <stdint.h>
 
-static unsigned int random_seed = 53455346;
-
-int isupper(int c) { return (c >= 65 && c <= 90); }
-
-int islower(int c) { return (c >= 97 && c <= 122); }
-
-int toupper(int c) {
-    if (!islower(c)) {
-        return c;
-    }
-
-    return c - 32;
+int isupper(int c)
+{
+	return (c >= 65 && c <= 90);
 }
 
-int tolower(int c) {
-    if (!isupper(c)) {
-        return c;
-    }
-
-    return c + 32;
+int islower(int c)
+{
+	return (c >= 97 && c <= 122);
 }
 
-int isdigit(int c) { return (c >= 48 && c <= 57); }
+int toupper(int c)
+{
+	if (!islower(c)) {
+		return c;
+	}
 
-int isalpha(int c) { return (c >= 65 && c <= 90) || (c >= 97 && c <= 122); }
-
-int isalnum(int c) { return isalpha(c) | isdigit(c); }
-
-int iscntrl(int c) { return (c <= 31) || (c == 127); }
-
-int isprint(int c) { return (c >= 32 && c <= 126) || (c >= 128 && c <= 254); }
-
-int isgraph(int c) { return isprint(c) && c != 32; }
-
-int ispunct(int c) { return isgraph(c) && !isalnum(c); }
-
-int isspace(int c) {
-    return (c == ' ') || (c == '\t') || (c == '\n') || (c == '\v') ||
-           (c == '\f') || (c == '\r');
+	return c - 32;
 }
 
-int isxdigit(int c) {
-    return isdigit(c) || (c >= 65 && c <= 70) || (c >= 97 && c <= 102);
+int tolower(int c)
+{
+	if (!isupper(c)) {
+		return c;
+	}
+
+	return c + 32;
 }
 
-bool fill_random(unsigned char *buffer, unsigned int size) {
-    if (!buffer || !size) {
-        return false;
-    }
-
-    for (uint32_t i = 0; i < size; i++) {
-        uint32_t next = random_seed;
-        uint32_t result;
-
-        next *= 1103515245;
-        next += 12345;
-        result = (uint32_t)(next / 65536) % 2048;
-
-        next *= 1103515245;
-        next += 12345;
-        result <<= 10;
-        result ^= (uint32_t)(next / 65536) % 1024;
-
-        next *= 1103515245;
-        next += 12345;
-        result <<= 10;
-        result ^= (uint32_t)(next / 65536) % 1024;
-        random_seed = next;
-
-        buffer[i] = (uint8_t)(result % 256);
-    }
-
-    return true;
+int isdigit(int c)
+{
+	return (c >= 48 && c <= 57);
+}
+
+int isalpha(int c)
+{
+	return (c >= 65 && c <= 90) || (c >= 97 && c <= 122);
+}
+
+int isalnum(int c)
+{
+	return isalpha(c) | isdigit(c);
+}
+
+int iscntrl(int c)
+{
+	return (c <= 31) || (c == 127);
+}
+
+int isprint(int c)
+{
+	return (c >= 32 && c <= 126) || (c >= 128 && c <= 254);
+}
+
+int isgraph(int c)
+{
+	return isprint(c) && c != 32;
+}
+
+int ispunct(int c)
+{
+	return isgraph(c) && !isalnum(c);
+}
+
+int isspace(int c)
+{
+	return (c == ' ') || (c == '\t') || (c == '\n') || (c == '\v')
+	    || (c == '\f') || (c == '\r');
+}
+
+int isxdigit(int c)
+{
+	return isdigit(c) || (c >= 65 && c <= 70) || (c >= 97 && c <= 102);
 }

libc/include/mango/libc/ctype.h

@@ -18,7 +18,6 @@ extern int isupper(int c);
 extern int isxdigit(int c);
 extern int tolower(int c);
 extern int toupper(int c);
-extern int fill_random(unsigned char *buffer, unsigned int size);
 
 #ifdef __cplusplus
 }

sched/core.c

@@ -1,9 +1,10 @@
-#include <mango/object.h>
-#include <mango/sched.h>
 #include <mango/clock.h>
 #include <mango/cpu.h>
-#include <mango/printk.h>
 #include <mango/machine/thread.h>
+#include <mango/object.h>
+#include <mango/printk.h>
+#include <mango/sched.h>
+#include <mango/vm-region.h>
 
 extern kern_status_t setup_kernel_task(void);
 extern kern_status_t setup_idle_task(void);
@@ -37,8 +38,14 @@ kern_status_t sched_init(void)
 		return status;
 	}
 
-	struct thread *this_thread = QUEUE_CONTAINER(struct thread, tr_threads, queue_first(&kernel_task()->t_threads));
-	struct thread *idle_thread = QUEUE_CONTAINER(struct thread, tr_threads, queue_first(&idle_task()->t_threads));
+	struct thread *this_thread = QUEUE_CONTAINER(
+		struct thread,
+		tr_parent_entry,
+		queue_first(&kernel_task()->t_threads));
+	struct thread *idle_thread = QUEUE_CONTAINER(
+		struct thread,
+		tr_parent_entry,
+		queue_first(&idle_task()->t_threads));
 
 	struct cpu_data *this_cpu = get_this_cpu();
 	rq_init(&this_cpu->c_rq);
@@ -55,7 +62,8 @@ kern_status_t sched_init(void)
 
 static void expire_timers(struct cpu_data *cpu)
 {
-	queue_foreach(struct timer, timer, &cpu->c_timers, t_entry) {
+	queue_foreach(struct timer, timer, &cpu->c_timers, t_entry)
+	{
 		if (timer->t_expiry <= clock_ticks) {
 			timer->t_callback(timer);
 		}
@@ -64,11 +72,21 @@ static void expire_timers(struct cpu_data *cpu)
 
 void context_switch(struct thread *old, struct thread *new)
 {
-	if (old->tr_parent->t_pmap != new->tr_parent->t_pmap) {
-		pmap_switch(new->tr_parent->t_pmap);
+	struct ml_cpu_block *this_cpu = ml_this_cpu();
+
+	old->tr_cpu_kernel_sp = ml_cpu_block_get_kstack(this_cpu);
+	old->tr_cpu_user_sp = ml_cpu_block_get_ustack(this_cpu);
+
+	pmap_t old_pmap = old->tr_parent->t_pmap;
+	pmap_t new_pmap = new->tr_parent->t_pmap;
+
+	if (old_pmap != new_pmap) {
+		pmap_switch(new_pmap);
 	}
 
-	switch_to(old, new);
+	ml_cpu_block_set_kstack(this_cpu, new->tr_cpu_kernel_sp);
+	ml_cpu_block_set_ustack(this_cpu, new->tr_cpu_user_sp);
+	ml_thread_switch(old, new);
 }
 
 void __schedule(enum sched_mode mode)
@@ -102,7 +120,8 @@ void __schedule(enum sched_mode mode)
 
 	enum thread_state prev_state = READ_ONCE(prev->tr_state);
 
-	if ((mode == SCHED_IRQ || prev_state == THREAD_READY) && prev != rq->rq_idle) {
+	if ((mode == SCHED_IRQ || prev_state == THREAD_READY)
+	    && prev != rq->rq_idle) {
 		rq_enqueue(rq, prev);
 	}
 
@@ -213,7 +232,8 @@ void end_charge_period(void)
 
 	self->tr_charge_period_start = 0;
 
-	//printk("%llu cycles charged to %s/%u", charge, self->tr_parent->t_name, self->tr_parent->t_id);
+	// printk("%llu cycles charged to %s/%u", charge,
+	// self->tr_parent->t_name, self->tr_parent->t_id);
 }
 
 cycles_t default_quantum(void)

sched/task.c

@@ -1,10 +1,12 @@
-#include <mango/locks.h>
-#include <mango/printk.h>
 #include <mango/clock.h>
-#include <mango/sched.h>
-#include <mango/object.h>
 #include <mango/cpu.h>
+#include <mango/handle.h>
 #include <mango/libc/stdio.h>
+#include <mango/locks.h>
+#include <mango/object.h>
+#include <mango/printk.h>
+#include <mango/sched.h>
+#include <mango/vm-region.h>
 
 #define TASK_CAST(p) OBJECT_C_CAST(struct task, t_base, &task_type, p)
 
@@ -17,10 +19,18 @@ static struct object_type task_type = {
 static struct task *__kernel_task;
 static struct task *__idle_task;
 
-static spin_lock_t task_list_lock;
+static spin_lock_t pid_map_lock = SPIN_LOCK_INIT;
+static DECLARE_BITMAP(pid_map, PID_MAX);
+
+static spin_lock_t task_list_lock = SPIN_LOCK_INIT;
 static struct btree task_list;
 
-BTREE_DEFINE_SIMPLE_GET(struct task, unsigned int, t_tasklist, t_id, task_list_get)
+BTREE_DEFINE_SIMPLE_GET(
+	struct task,
+	unsigned int,
+	t_tasklist,
+	t_id,
+	task_list_get)
 BTREE_DEFINE_SIMPLE_INSERT(struct task, t_tasklist, t_id, task_list_insert)
 
 struct task *kernel_task(void)
@@ -40,6 +50,31 @@ void idle(void)
 	}
 }
 
+static unsigned int pid_alloc(void)
+{
+	unsigned long flags;
+	spin_lock_irqsave(&pid_map_lock, &flags);
+
+	unsigned int pid = bitmap_lowest_clear(pid_map, PID_MAX);
+	if (pid != BITMAP_NPOS) {
+		bitmap_set(pid_map, pid);
+	}
+
+	spin_unlock_irqrestore(&pid_map_lock, flags);
+
+	return pid;
+}
+
+static void pid_free(unsigned int pid)
+{
+	unsigned long flags;
+	spin_lock_irqsave(&pid_map_lock, &flags);
+
+	bitmap_clear(pid_map, pid);
+
+	spin_unlock_irqrestore(&pid_map_lock, flags);
+}
+
 kern_status_t setup_kernel_task(void)
 {
 	__kernel_task = task_alloc();
@@ -47,11 +82,22 @@ kern_status_t setup_kernel_task(void)
 		return KERN_NO_MEMORY;
 	}
 
-	__kernel_task->t_id = 0;
-	__kernel_task->t_pmap = get_kernel_pmap();
+	__kernel_task->t_id = -1;
 	__kernel_task->t_state = TASK_RUNNING;
+	__kernel_task->t_pmap = get_kernel_pmap();
 
-	snprintf(__kernel_task->t_name, sizeof __kernel_task->t_name, "kernel_task");
+	vm_region_create(
+		NULL,
+		"root",
+		VM_KERNEL_BASE,
+		VM_KERNEL_LIMIT - VM_KERNEL_BASE,
+		VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXEC | VM_PROT_SVR,
+		&__kernel_task->t_address_space);
+
+	snprintf(
+		__kernel_task->t_name,
+		sizeof __kernel_task->t_name,
+		"kernel_task");
 
 	struct thread *kernel_thread = thread_alloc();
 	kernel_thread->tr_id = 0;
@@ -62,7 +108,9 @@ kern_status_t setup_kernel_task(void)
 
 	unsigned long flags;
 	task_lock_irqsave(__kernel_task, &flags);
-	queue_push_back(&__kernel_task->t_threads, &kernel_thread->tr_threads);
+	queue_push_back(
+		&__kernel_task->t_threads,
+		&kernel_thread->tr_parent_entry);
 	task_unlock_irqrestore(__kernel_task, flags);
 
 	spin_lock_irqsave(&task_list_lock, &flags);
@@ -82,24 +130,24 @@ kern_status_t setup_idle_task(void)
 	unsigned long flags;
 	task_lock_irqsave(__idle_task, &flags);
 
-	__idle_task->t_id = (unsigned int)-1;
-	__idle_task->t_pmap = get_kernel_pmap();
+	__idle_task->t_id = -2;
 	__idle_task->t_state = TASK_RUNNING;
+	__idle_task->t_pmap = get_kernel_pmap();
 
 	snprintf(__idle_task->t_name, sizeof __idle_task->t_name, "idle");
 
 	struct thread *idle_thread = thread_alloc();
 	if (!idle_thread) {
-		task_deref(__idle_task);
+		task_unref(__idle_task);
 		__idle_task = NULL;
 		return KERN_NO_MEMORY;
 	}
 
 	idle_thread->tr_id = 0;
 	idle_thread->tr_parent = __idle_task;
-	thread_init(idle_thread, (uintptr_t)idle);
+	thread_init_kernel(idle_thread, (uintptr_t)idle);
 
-	queue_push_back(&__idle_task->t_threads, &idle_thread->tr_threads);
+	queue_push_back(&__idle_task->t_threads, &idle_thread->tr_parent_entry);
 
 	task_unlock_irqrestore(__idle_task, flags);
 
@@ -119,10 +167,53 @@ struct task *task_alloc(void)
 	}
 
 	struct task *t = TASK_CAST(task_obj);
-	memset(t, 0x00, sizeof *t);
 	return t;
 }
 
+struct task *task_create(struct task *parent, const char *name)
+{
+	struct task *task = task_alloc();
+	if (!task) {
+		return NULL;
+	}
+
+	pmap_t pmap = pmap_create();
+	if (pmap == PMAP_INVALID) {
+		object_unref(&task->t_base);
+		return NULL;
+	}
+
+	task->t_id = pid_alloc();
+	task->t_pmap = pmap;
+	vm_region_create(
+		NULL,
+		"root",
+		VM_USER_BASE,
+		VM_USER_LIMIT - VM_USER_BASE,
+		VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXEC | VM_PROT_USER,
+		&task->t_address_space);
+
+	task->t_address_space->vr_pmap = pmap;
+	task->t_state = TASK_STOPPED;
+	task->t_handles = handle_table_create();
+
+	if (name) {
+		strncpy(task->t_name, name, sizeof task->t_name);
+		task->t_name[sizeof task->t_name - 1] = '\0';
+	}
+
+	unsigned long flags;
+	task_lock_irqsave(parent, &flags);
+	queue_push_back(&parent->t_children, &task->t_child_entry);
+	task_unlock_irqrestore(parent, flags);
+
+	spin_lock_irqsave(&task_list_lock, &flags);
+	task_list_insert(&task_list, task);
+	spin_unlock_irqrestore(&task_list_lock, flags);
+
+	return task;
+}
+
 struct task *task_from_pid(unsigned int pid)
 {
 	unsigned long flags;
@@ -132,6 +223,20 @@ struct task *task_from_pid(unsigned int pid)
 	return t;
 }
 
+struct thread *task_create_thread(struct task *parent)
+{
+	struct thread *thread = thread_alloc();
+	thread->tr_id = parent->t_next_thread_id++;
+	thread->tr_prio = PRIO_NORMAL;
+	thread->tr_state = THREAD_STOPPED;
+	thread->tr_parent = parent;
+	thread->tr_quantum_target = default_quantum();
+
+	queue_push_back(&parent->t_threads, &thread->tr_parent_entry);
+
+	return thread;
+}
+
 struct task *current_task(void)
 {
 	struct thread *thr = current_thread();

sched/thread.c

@@ -6,8 +6,6 @@
 
 #define THREAD_CAST(p) OBJECT_C_CAST(struct thread, thr_base, &thread_type, p)
 
-static DECLARE_BITMAP(thread_ids, THREAD_MAX) = {0};
-
 static struct object_type thread_type = {
 	.ob_name = "thread",
 	.ob_size = sizeof(struct thread),
@@ -31,10 +29,9 @@ struct thread *thread_alloc(void)
 	return t;
 }
 
-kern_status_t thread_init(struct thread *thr, uintptr_t ip)
+kern_status_t thread_init_kernel(struct thread *thr, virt_addr_t ip)
 {
-	thr->tr_id = bitmap_lowest_clear(thread_ids, THREAD_MAX);
-	bitmap_set(thread_ids, thr->tr_id);
+	thr->tr_id = thr->tr_parent->t_next_thread_id++;
 
 	thr->tr_prio = PRIO_NORMAL;
 	thr->tr_state = THREAD_READY;
@@ -49,7 +46,51 @@ kern_status_t thread_init(struct thread *thr, uintptr_t ip)
 	           + vm_page_order_to_bytes(THREAD_KSTACK_ORDER);
 	thr->tr_bp = thr->tr_sp;
 
-	prepare_stack(ip, &thr->tr_sp);
+	ml_thread_prepare_kernel_context(ip, &thr->tr_sp);
+
+	return KERN_OK;
+}
+
+kern_status_t thread_init_user(
+	struct thread *thr,
+	virt_addr_t ip,
+	virt_addr_t sp)
+{
+	thr->tr_id = thr->tr_parent->t_next_thread_id++;
+
+	thr->tr_prio = PRIO_NORMAL;
+	thr->tr_state = THREAD_READY;
+	thr->tr_quantum_target = default_quantum();
+
+	thr->tr_kstack = vm_page_alloc(THREAD_KSTACK_ORDER, VM_NORMAL);
+	if (!thr->tr_kstack) {
+		return KERN_NO_MEMORY;
+	}
+
+	thr->tr_sp = (uintptr_t)vm_page_get_vaddr(thr->tr_kstack)
+	           + vm_page_order_to_bytes(THREAD_KSTACK_ORDER);
+	thr->tr_bp = thr->tr_sp;
+	thr->tr_cpu_kernel_sp = thr->tr_sp;
+
+	/* the new thread needs two contextx:
+	 *   1) to get the thread running in kernel mode, so that it can
+	 *   execute ml_thread_switch_user
+	 *   2) to allow ml_thread_switch_user to jump to the correct place
+	 *   in usermode (and with the correct stack).
+	 *
+	 * these two contexts are constructed on the thread's kernel stack
+	 * in reverse order.
+	 */
+
+	/* this context will be used by ml_user_return to jump to userspace
+	 * with the specified instruction pointer and user stack */
+	ml_thread_prepare_user_context(ip, sp, &thr->tr_sp);
+	/* this context will be used by the scheduler and ml_thread_switch to
+	 * jump to ml_user_return in kernel mode with the thread's kernel stack.
+	 */
+	ml_thread_prepare_kernel_context(
+		(uintptr_t)ml_thread_switch_user,
+		&thr->tr_sp);
 
 	return KERN_OK;
 }
@@ -85,19 +126,18 @@ struct thread *create_kernel_thread(void (*fn)(void))
 	struct task *kernel = kernel_task();
 	struct thread *thr = thread_alloc();
 
-	thr->tr_id = 1;
-	bitmap_set(thread_ids, 1);
+	thr->tr_id = kernel->t_next_thread_id++;
 
 	thr->tr_parent = kernel;
 	thr->tr_prio = PRIO_NORMAL;
 	thr->tr_state = THREAD_READY;
 	thr->tr_quantum_target = default_quantum();
 
-	thread_init(thr, (uintptr_t)fn);
+	thread_init_kernel(thr, (uintptr_t)fn);
 
 	unsigned long flags;
 	task_lock_irqsave(kernel, &flags);
-	queue_push_back(&kernel->t_threads, &thr->tr_threads);
+	queue_push_back(&kernel->t_threads, &thr->tr_parent_entry);
 	task_unlock_irqrestore(kernel, flags);
 
 	schedule_thread_on_cpu(thr);
@@ -110,8 +150,7 @@ struct thread *create_idle_thread(void)
 	struct task *idle = idle_task();
 	struct thread *thr = thread_alloc();
 
-	thr->tr_id = 0;
-	bitmap_set(thread_ids, 0);
+	thr->tr_id = idle->t_next_thread_id++;
 
 	thr->tr_parent = idle;
 	thr->tr_prio = PRIO_NORMAL;
@@ -120,7 +159,7 @@ struct thread *create_idle_thread(void)
 
 	unsigned long flags;
 	task_lock_irqsave(idle, &flags);
-	queue_push_back(&idle->t_threads, &thr->tr_threads);
+	queue_push_back(&idle->t_threads, &thr->tr_parent_entry);
 	task_unlock_irqrestore(idle, flags);
 
 	return thr;

test/obj.c

@@ -53,7 +53,7 @@ static void print_object_tree(struct object *obj, int depth)
 		}
 
 		print_object_tree(child, depth + 1);
-		object_deref(child);
+		object_unref(child);
 		i++;
 	}
 }

util/random.c

@@ -0,0 +1,36 @@
+#include <mango/util.h>
+
+static unsigned int random_seed = 53455346;
+
+bool fill_random(void *p, unsigned int size)
+{
+	unsigned char *buffer = p;
+
+	if (!buffer || !size) {
+		return false;
+	}
+
+	for (uint32_t i = 0; i < size; i++) {
+		uint32_t next = random_seed;
+		uint32_t result;
+
+		next *= 1103515245;
+		next += 12345;
+		result = (uint32_t)(next / 65536) % 2048;
+
+		next *= 1103515245;
+		next += 12345;
+		result <<= 10;
+		result ^= (uint32_t)(next / 65536) % 1024;
+
+		next *= 1103515245;
+		next += 12345;
+		result <<= 10;
+		result ^= (uint32_t)(next / 65536) % 1024;
+		random_seed = next;
+
+		buffer[i] = (uint8_t)(result % 256);
+	}
+
+	return true;
+}

vm/bootstrap.c

@@ -1,17 +1,20 @@
-#include <mango/status.h>
 #include <limits.h>
-#include <mango/vm.h>
+#include <mango/machine/cpu.h>
 #include <mango/memblock.h>
 #include <mango/printk.h>
-#include <mango/machine/cpu.h>
+#include <mango/status.h>
+#include <mango/vm-object.h>
+#include <mango/vm-region.h>
+#include <mango/vm.h>
 #include <stddef.h>
-#include <limits.h>
 #include <stdint.h>
 
 /* One struct vm_pg_data per NUMA node. */
 static struct vm_pg_data *node_data = NULL;
 
-kern_status_t vm_bootstrap(const struct vm_zone_descriptor *zones, size_t nr_zones)
+kern_status_t vm_bootstrap(
+	const struct vm_zone_descriptor *zones,
+	size_t nr_zones)
 {
 	int numa_count = 1;
 
@@ -34,10 +37,12 @@ kern_status_t vm_bootstrap(const struct vm_zone_descriptor *zones, size_t nr_zon
 	}
 
 	for (size_t i = 0; i < nr_zones; i++) {
-		vm_zone_init(&node_data->pg_zones[zones[i].zd_id],  &zones[i]);
+		vm_zone_init(&node_data->pg_zones[zones[i].zd_id], &zones[i]);
 	}
 
 	kmalloc_init();
+	vm_object_type_init();
+	vm_region_type_init();
 	return KERN_OK;
 }
 

vm/memblock.c

@@ -19,27 +19,29 @@
     contributors may be used to endorse or promote products derived from this
     software without specific prior written permission.
  */
-#include <stdbool.h>
 #include <limits.h>
-#include <mango/types.h>
 #include <mango/libc/string.h>
 #include <mango/memblock.h>
+#include <mango/types.h>
+#include <stdbool.h>
 
-#define MIN(a, b) ((a) < (b) ? (a) : (b))
-#define MAX(a, b) ((a) > (b) ? (a) : (b))
+#define MIN(a, b)  ((a) < (b) ? (a) : (b))
+#define MAX(a, b)  ((a) > (b) ? (a) : (b))
 
 #define ITER(a, b) ((uint64_t)(a) | ((uint64_t)(b) << 32))
-#define ITER_END ULLONG_MAX
+#define ITER_END   ULLONG_MAX
 #define IDX_A(idx) ((idx) & 0xFFFFFFFF)
 #define IDX_B(idx) (((idx) >> 32) & 0xFFFFFFFF)
 
 /* the maximum possible value for a pointer type.
    Note that any pointers returned by the memblock API will still
    be bounded by the defined memory regions, and not by this constant. */
-#define ADDR_MAX (~(uintptr_t)0)
+#define ADDR_MAX   (~(uintptr_t)0)
 
-static struct memblock_region init_memory_regions[MEMBLOCK_INIT_MEMORY_REGION_COUNT];
-static struct memblock_region init_reserved_regions[MEMBLOCK_INIT_RESERVED_REGION_COUNT];
+static struct memblock_region
+	init_memory_regions[MEMBLOCK_INIT_MEMORY_REGION_COUNT];
+static struct memblock_region
+	init_reserved_regions[MEMBLOCK_INIT_RESERVED_REGION_COUNT];
 
 static phys_addr_t do_alloc(size_t size, phys_addr_t align);
 
@@ -59,16 +61,21 @@ static void memblock_double_capacity(struct memblock_type *type)
 {
 	size_t new_max = type->max * 2;
 
-	phys_addr_t new_regions_p = do_alloc(new_max * sizeof(struct memblock_region), 8);
+	phys_addr_t new_regions_p
+		= do_alloc(new_max * sizeof(struct memblock_region), 8);
 
 	void *new_regions = (void *)(new_regions_p + memblock.m_voffset);
-	memcpy(new_regions, type->regions, type->count * sizeof(struct memblock_region));
+	memcpy(new_regions,
+	       type->regions,
+	       type->count * sizeof(struct memblock_region));
 
 	type->regions = new_regions;
 	type->max = new_max;
 }
 
-static int memblock_insert_region(struct memblock_type *type, struct memblock_region *to_add)
+static int memblock_insert_region(
+	struct memblock_type *type,
+	struct memblock_region *to_add)
 {
 	unsigned int i = 0;
 
@@ -110,13 +117,17 @@ static int memblock_remove_region(struct memblock_type *type, unsigned int i)
 int memblock_init(uintptr_t alloc_start, uintptr_t alloc_end, uintptr_t voffset)
 {
 	memblock.m_alloc_start = alloc_start;
-	memblock.m_alloc_end =alloc_end;
+	memblock.m_alloc_end = alloc_end;
 	memblock.m_voffset = voffset;
 
 	return 0;
 }
 
-int memblock_add_range(struct memblock_type *type, uintptr_t base, size_t size, enum memblock_region_status status)
+int memblock_add_range(
+	struct memblock_type *type,
+	uintptr_t base,
+	size_t size,
+	enum memblock_region_status status)
 {
 	if (size == 0) {
 		return 0;
@@ -131,14 +142,17 @@ int memblock_add_range(struct memblock_type *type, uintptr_t base, size_t size,
 		return 0;
 	}
 
-	struct memblock_region new_region = { .base = base, .limit = limit, .status = status };
+	struct memblock_region new_region
+		= {.base = base, .limit = limit, .status = status};
 
-	/* two regions with different statuses CANNOT intersect. we first need to check
-	   to make sure the region being added doesn't violate this rule. */
+	/* two regions with different statuses CANNOT intersect. we first need
+	   to check to make sure the region being added doesn't violate this
+	   rule. */
 	for (unsigned int i = 0; i < type->count; i++) {
 		struct memblock_region *cur_region = &type->regions[i];
 
-		if (new_region.base > cur_region->limit || new_region.limit < cur_region->base) {
+		if (new_region.base > cur_region->limit
+		    || new_region.limit < cur_region->base) {
 			continue;
 		}
 
@@ -154,47 +168,67 @@ int memblock_add_range(struct memblock_type *type, uintptr_t base, size_t size,
 	for (unsigned int i = 0; i < type->count; i++) {
 		struct memblock_region *cur_region = &type->regions[i];
 
-		/* case 1: the region being added and the current region have no connection what-so-ever (no overlaps) */
-		if (cur_region->limit + 1 < new_region.base || cur_region->base > new_region.limit) {
+		/* case 1: the region being added and the current region have no
+		 * connection what-so-ever (no overlaps) */
+		if (cur_region->limit + 1 < new_region.base
+		    || cur_region->base > new_region.limit) {
 			continue;
 		}
 
-		/* case 2: the region being added matches a region already in the list. */
-		if (cur_region->base == new_region.base && cur_region->limit == new_region.limit) {
+		/* case 2: the region being added matches a region already in
+		 * the list. */
+		if (cur_region->base == new_region.base
+		    && cur_region->limit == new_region.limit) {
 			/* nothing needs to be done */
 			add_new = false;
 			break;
 		}
 
-
-		/* case 3: the region being added completely contains a region already in the list. */
-		if (cur_region->base > new_region.base && cur_region->limit <= new_region.limit) {
+		/* case 3: the region being added completely contains a region
+		 * already in the list. */
+		if (cur_region->base > new_region.base
+		    && cur_region->limit <= new_region.limit) {
 			memblock_remove_region(type, i);
 
-			/* after memblock_remove_region(), a different region will have moved into the array slot referenced by i.
-			   decrementing i means we'll stay at the current index and process this region. */
+			/* after memblock_remove_region(), a different region
+			   will have moved into the array slot referenced by i.
+			   decrementing i means we'll stay at the current index
+			   and process this region. */
 			i--;
 			continue;
 		}
 
+		/* case 4: the region being added meets or partially overlaps a
+		 * region already in the list. */
 
-		/* case 4: the region being added meets or partially overlaps a region already in the list. */
-
-		/* there can be an overlap at the beginning and the end of the region being added,
-		   anything else is either a full overlap (case 3) or not within the region being added at all.
-		   to handle this, remove the region that's already in the list and extend the region being added to cover it.
-		   the two regions may overlap and have incompatible statuses, but this case was handled earlier in this function. */
-		if ((new_region.base > cur_region->base || new_region.base == cur_region->limit - 1) && new_region.status == cur_region->status) {
-			/* the new region overlaps the END of the current region, change the base of the new region to match that of the current region. */
+		/* there can be an overlap at the beginning and the end of the
+		   region being added, anything else is either a full overlap
+		   (case 3) or not within the region being added at all. to
+		   handle this, remove the region that's already in the list and
+		   extend the region being added to cover it. the two regions
+		   may overlap and have incompatible statuses, but this case was
+		   handled earlier in this function. */
+		if ((new_region.base > cur_region->base
+		     || new_region.base == cur_region->limit - 1)
+		    && new_region.status == cur_region->status) {
+			/* the new region overlaps the END of the current
+			 * region, change the base of the new region to match
+			 * that of the current region. */
 			new_region.base = cur_region->base;
-		} else if ((new_region.base < cur_region->base || new_region.limit + 1 == cur_region->base) && new_region.status == cur_region->status) {
-			/* the new region overlaps the BEGINNING of the current region, change the limit of the new region to match that of the current region. */
+		} else if (
+			(new_region.base < cur_region->base
+		         || new_region.limit + 1 == cur_region->base)
+			&& new_region.status == cur_region->status) {
+			/* the new region overlaps the BEGINNING of the current
+			 * region, change the limit of the new region to match
+			 * that of the current region. */
 			new_region.limit = cur_region->limit;
 		} else {
 			continue;
 		}
 
-		/* with the new region updated to include the current region, we can remove the current region from the list */
+		/* with the new region updated to include the current region, we
+		 * can remove the current region from the list */
 		memblock_remove_region(type, i);
 		i--;
 	}
@@ -216,7 +250,11 @@ int memblock_add(uintptr_t base, size_t size)
 		memblock_double_capacity(&memblock.memory);
 	}
 
-	return memblock_add_range(&memblock.memory, base, size, MEMBLOCK_MEMORY);
+	return memblock_add_range(
+		&memblock.memory,
+		base,
+		size,
+		MEMBLOCK_MEMORY);
 }
 
 int memblock_reserve(uintptr_t base, size_t size)
@@ -225,7 +263,11 @@ int memblock_reserve(uintptr_t base, size_t size)
 		memblock_double_capacity(&memblock.reserved);
 	}
 
-	return memblock_add_range(&memblock.reserved, base, size, MEMBLOCK_RESERVED);
+	return memblock_add_range(
+		&memblock.reserved,
+		base,
+		size,
+		MEMBLOCK_RESERVED);
 }
 
 static phys_addr_t do_alloc(size_t size, phys_addr_t align)
@@ -245,7 +287,8 @@ static phys_addr_t do_alloc(size_t size, phys_addr_t align)
 	phys_addr_t region_end = memblock.m_alloc_end - memblock.m_voffset;
 
 	struct memblock_iter it;
-	for_each_free_mem_range (&it, region_start, region_end) {
+	for_each_free_mem_range(&it, region_start, region_end)
+	{
 		phys_addr_t base = it.it_base;
 		if (base & (align - 1)) {
 			base &= ~(align - 1);
@@ -270,7 +313,11 @@ static phys_addr_t do_alloc(size_t size, phys_addr_t align)
 		return 0;
 	}
 
-	int status = memblock_add_range(&memblock.reserved, allocated_base, allocated_limit - allocated_base, MEMBLOCK_ALLOC);
+	int status = memblock_add_range(
+		&memblock.reserved,
+		allocated_base,
+		allocated_limit - allocated_base,
+		MEMBLOCK_ALLOC);
 	if (status != 0) {
 		return 0;
 	}
@@ -312,9 +359,11 @@ int memblock_free_phys(phys_addr_t addr, size_t size)
 }
 
 void __next_memory_region(
-		struct memblock_iter *it,
-		struct memblock_type *type_a, struct memblock_type *type_b,
-		uintptr_t start, uintptr_t end)
+	struct memblock_iter *it,
+	struct memblock_type *type_a,
+	struct memblock_type *type_b,
+	uintptr_t start,
+	uintptr_t end)
 {
 	unsigned int idx_a = IDX_A(it->__idx);
 	unsigned int idx_b = IDX_B(it->__idx);
@@ -344,70 +393,85 @@ void __next_memory_region(
 		}
 
 		if (m_start > end) {
-			/* we have gone past the requested memory range and can now stop */
+			/* we have gone past the requested memory range and can
+			 * now stop */
 			break;
 		}
 
 		for (; idx_b < type_b->count + 1; idx_b++) {
 			struct memblock_region *r = &type_b->regions[idx_b];
 
-			/* r_start and r_end delimit the region of memory between the current and previous reserved regions.
-			   if we have gone past the last reserved region, these variables delimit the range between the end
-			   of the last reserved region and the end of memory. */
+			/* r_start and r_end delimit the region of memory
+			   between the current and previous reserved regions. if
+			   we have gone past the last reserved region, these
+			   variables delimit the range between the end of the
+			   last reserved region and the end of memory. */
 			uintptr_t r_start = idx_b > 0 ? r[-1].limit + 1 : 0;
 			uintptr_t r_end;
 
 			if (idx_b < type_b->count) {
 				r_end = r->base;
 
-				/* we decrement r_end to get the address of the last byte of the free region.
-				   if r_end is already zero, there is a reserved region starting at address 0x0.
-				   as long as r_end == r_start == 0x00000, we will skip this region. */
+				/* we decrement r_end to get the address of the
+				   last byte of the free region. if r_end is
+				   already zero, there is a reserved region
+				   starting at address 0x0. as long as r_end ==
+				   r_start == 0x00000, we will skip this region.
+				 */
 				if (r_end) {
 					r_end--;
 				}
 			} else {
-				/* this maximum value will be clamped to the bounds of memblock.memory
-				   before being returned to the caller */
+				/* this maximum value will be clamped to the
+				   bounds of memblock.memory before being
+				   returned to the caller */
 				r_end = ADDR_MAX;
 			}
 
 			if (r_start >= r_end) {
-				/* this free region has a length of zero, move to the next one */
+				/* this free region has a length of zero, move
+				 * to the next one */
 				continue;
 			}
 
 			if (r_start >= m_end) {
-				/* we've gone past the end of the current memory region, and need to go to the next one */
+				/* we've gone past the end of the current memory
+				 * region, and need to go to the next one */
 				break;
 			}
 
-			/* we've already gone past this free memory region. move to the next one */
+			/* we've already gone past this free memory region. move
+			 * to the next one */
 			if (m_start >= r_end) {
 				continue;
 			}
 
 			/* we want the area that is overlapped by both
-				region M (m_start - m_end) : The region defined as system memory.
-				region R (r_start - r_end) : The region defined as free / outside of any reserved regions.
+			        region M (m_start - m_end) : The region defined
+			   as system memory. region R (r_start - r_end) : The
+			   region defined as free / outside of any reserved
+			   regions.
 			 */
 			it->it_base = MAX(m_start, r_start);
 			it->it_limit = MIN(m_end, r_end);
 
-			/* further limit the region to the intersection between the region itself and the
-			   specified iteration bounds */
+			/* further limit the region to the intersection between
+			   the region itself and the specified iteration bounds
+			 */
 			it->it_base = MAX(it->it_base, start);
 			it->it_limit = MIN(it->it_limit, end);
 
 			if (it->it_limit <= it->it_base) {
-				/* this region is not part of the specified bounds, skip it. */
+				/* this region is not part of the specified
+				 * bounds, skip it. */
 				continue;
 			}
 
 			it->it_status = MEMBLOCK_MEMORY;
 
-			/* whichever region is smaller, increment the pointer for that type, so we can
-			   compare the larger region with the next region of the incremented type. */
+			/* whichever region is smaller, increment the pointer
+			   for that type, so we can compare the larger region
+			   with the next region of the incremented type. */
 			if (m_end <= r_end) {
 				idx_a++;
 			} else {
@@ -424,7 +488,7 @@ void __next_memory_region(
 	it->__idx = ITER_END;
 }
 
-phys_addr_t memblock_virt_to_phys(void *p)
+phys_addr_t memblock_virt_to_phys(const void *p)
 {
 	return (phys_addr_t)p - memblock.m_voffset;
 }

vm/page.c

@@ -1,44 +1,44 @@
-#include <mango/types.h>
+#include <mango/libc/string.h>
 #include <mango/memblock.h>
 #include <mango/printk.h>
+#include <mango/types.h>
 #include <mango/vm.h>
-#include <mango/libc/string.h>
 
 /* Pre-calculated page order -> size conversion table */
 static size_t page_order_bytes[] = {
-	[VM_PAGE_4K]   = 0x1000,
-	[VM_PAGE_8K]   = 0x2000,
-	[VM_PAGE_16K]  = 0x4000,
-	[VM_PAGE_32K]  = 0x8000,
-	[VM_PAGE_64K]  = 0x10000,
+	[VM_PAGE_4K] = 0x1000,
+	[VM_PAGE_8K] = 0x2000,
+	[VM_PAGE_16K] = 0x4000,
+	[VM_PAGE_32K] = 0x8000,
+	[VM_PAGE_64K] = 0x10000,
 	[VM_PAGE_128K] = 0x20000,
 	[VM_PAGE_256K] = 0x40000,
 	[VM_PAGE_512K] = 0x80000,
-	[VM_PAGE_1M]   = 0x100000,
-	[VM_PAGE_2M]   = 0x200000,
-	[VM_PAGE_4M]   = 0x400000,
-	[VM_PAGE_8M]   = 0x800000,
-	[VM_PAGE_16M]  = 0x1000000,
-	[VM_PAGE_32M]  = 0x2000000,
-	[VM_PAGE_64M]  = 0x4000000,
+	[VM_PAGE_1M] = 0x100000,
+	[VM_PAGE_2M] = 0x200000,
+	[VM_PAGE_4M] = 0x400000,
+	[VM_PAGE_8M] = 0x800000,
+	[VM_PAGE_16M] = 0x1000000,
+	[VM_PAGE_32M] = 0x2000000,
+	[VM_PAGE_64M] = 0x4000000,
 	[VM_PAGE_128M] = 0x8000000,
 
 	/* vm can support pages of this size, but
-	   struct vm_page only has 4 bits with which to store
-	   the page order, which cannot accomodate these
-	   larger order numbers */
+           struct vm_page only has 4 bits with which to store
+           the page order, which cannot accomodate these
+           larger order numbers */
 	[VM_PAGE_256M] = 0x10000000,
 	[VM_PAGE_512M] = 0x20000000,
-	[VM_PAGE_1G]   = 0x40000000,
-	[VM_PAGE_2G]   = 0x80000000,
-	[VM_PAGE_4G]   = 0x100000000,
-	[VM_PAGE_8G]   = 0x200000000,
-	[VM_PAGE_16G]  = 0x400000000,
-	[VM_PAGE_32G]  = 0x800000000,
-	[VM_PAGE_64G]  = 0x1000000000,
+	[VM_PAGE_1G] = 0x40000000,
+	[VM_PAGE_2G] = 0x80000000,
+	[VM_PAGE_4G] = 0x100000000,
+	[VM_PAGE_8G] = 0x200000000,
+	[VM_PAGE_16G] = 0x400000000,
+	[VM_PAGE_32G] = 0x800000000,
+	[VM_PAGE_64G] = 0x1000000000,
 };
 
-phys_addr_t vm_virt_to_phys(void *p)
+phys_addr_t vm_virt_to_phys(const void *p)
 {
 	uintptr_t pv = (uintptr_t)p;
 
@@ -56,7 +56,8 @@ phys_addr_t vm_virt_to_phys(void *p)
 
 void *vm_phys_to_virt(phys_addr_t p)
 {
-	if (p >= (memblock.m_alloc_start - memblock.m_voffset) && p < (memblock.m_alloc_end - memblock.m_voffset)) {
+	if (p >= (memblock.m_alloc_start - memblock.m_voffset)
+	    && p < (memblock.m_alloc_end - memblock.m_voffset)) {
 		return memblock_phys_to_virt(p);
 	}
 
@@ -124,11 +125,10 @@ vm_alignment_t vm_page_order_to_alignment(enum vm_page_order order)
 	return ~(page_order_bytes[order] - 1);
 }
 
-
 size_t vm_bytes_to_pages(size_t bytes)
 {
-	if (bytes & (VM_PAGE_SIZE-1)) {
-		bytes &= ~(VM_PAGE_SIZE-1);
+	if (bytes & (VM_PAGE_SIZE - 1)) {
+		bytes &= ~(VM_PAGE_SIZE - 1);
 		bytes += VM_PAGE_SIZE;
 	}
 
@@ -150,7 +150,6 @@ struct vm_zone *vm_page_get_zone(struct vm_page *pg)
 	return &node->pg_zones[pg->p_zone];
 }
 
-
 struct vm_page *vm_page_alloc(enum vm_page_order order, enum vm_flags flags)
 {
 	/* TODO prefer nodes closer to us */
@@ -232,7 +231,8 @@ struct vm_page *vm_page_merge(struct vm_page *a, struct vm_page *b)
 		return NULL;
 	}
 
-	if ((a->p_flags & (VM_PAGE_ALLOC | VM_PAGE_RESERVED)) != (b->p_flags & (VM_PAGE_ALLOC | VM_PAGE_RESERVED))) {
+	if ((a->p_flags & (VM_PAGE_ALLOC | VM_PAGE_RESERVED))
+	    != (b->p_flags & (VM_PAGE_ALLOC | VM_PAGE_RESERVED))) {
 		return NULL;
 	}
 

vm/sparse.c

@@ -22,13 +22,14 @@
    of the sparse memory model may be outweighed by the extra
    overhead, and the flat memory model may be a better choice.
  */
-#include <mango/vm.h>
 #include <mango/arg.h>
-#include <mango/printk.h>
-#include <mango/panic.h>
-#include <mango/memblock.h>
-#include <mango/util.h>
+#include <mango/bsp.h>
 #include <mango/machine/cpu.h>
+#include <mango/memblock.h>
+#include <mango/panic.h>
+#include <mango/printk.h>
+#include <mango/util.h>
+#include <mango/vm.h>
 
 static struct vm_sector *sector_array = NULL;
 static size_t sector_array_count = 0;
@@ -53,11 +54,16 @@ static enum sector_coverage_mode get_sector_coverage_mode(void)
 		return SECTOR_COVERAGE_ALL;
 	}
 
-	printk("vm: [sparse] ignoring unknown sector coverage mode '%s', using FREE", arg);
+	printk("vm: [sparse] ignoring unknown sector coverage mode '%s', using "
+	       "FREE",
+	       arg);
 	return SECTOR_COVERAGE_FREE;
 }
 
-static struct vm_sector *phys_addr_to_sector_and_index(phys_addr_t addr, size_t *sector_id, size_t *index)
+static struct vm_sector *phys_addr_to_sector_and_index(
+	phys_addr_t addr,
+	size_t *sector_id,
+	size_t *index)
 {
 	/* all sectors have the same size */
 	size_t step = vm_page_order_to_bytes(sector_array[0].s_size);
@@ -98,7 +104,6 @@ static struct vm_page *get_or_create_page(phys_addr_t addr)
 		}
 	}
 
-
 	sector->s_pages[page_number].p_sector = sector_number;
 	return &sector->s_pages[page_number];
 }
@@ -123,9 +128,13 @@ static enum vm_page_order find_minimum_sector_size(phys_addr_t pmem_end)
    are in need of improvement to ensure that sparse works well on a wide
    range of systems. */
 static void calculate_sector_size_and_count(
-		size_t last_reserved_pfn, size_t last_free_pfn, size_t limit_pfn,
-		size_t reserved_size, size_t free_size,
-		unsigned int *out_sector_count, enum vm_page_order *out_sector_size)
+	size_t last_reserved_pfn,
+	size_t last_free_pfn,
+	size_t limit_pfn,
+	size_t reserved_size,
+	size_t free_size,
+	unsigned int *out_sector_count,
+	enum vm_page_order *out_sector_size)
 {
 	/* we can support up to VM_MAX_SECTORS memory sectors.
 	   the minimum sector size is what ever is required
@@ -154,8 +163,8 @@ static void calculate_sector_size_and_count(
 		   threshold. */
 		sector_size++;
 
-		/* if the difference is particularly big, increase the sector size
-		   even further */
+		/* if the difference is particularly big, increase the sector
+		   size even further */
 		if (memdiff >= 0x1000000) {
 			sector_size++;
 		}
@@ -183,13 +192,15 @@ void vm_sparse_init(void)
 	size_t last_reserved_pfn = 0, last_free_pfn = 0;
 
 	struct memblock_iter it;
-	for_each_mem_range (&it, 0x0, UINTPTR_MAX) {
+	for_each_mem_range(&it, 0x0, UINTPTR_MAX)
+	{
 		if (pmem_limit < it.it_limit + 1) {
 			pmem_limit = it.it_limit + 1;
 		}
 	}
 
-	for_each_free_mem_range (&it, 0x0, UINTPTR_MAX) {
+	for_each_free_mem_range(&it, 0x0, UINTPTR_MAX)
+	{
 		free_size += it.it_limit - it.it_base + 1;
 
 		size_t last_pfn = it.it_limit / VM_PAGE_SIZE;
@@ -199,7 +210,8 @@ void vm_sparse_init(void)
 		}
 	}
 
-	for_each_reserved_mem_range (&it, 0x0, UINTPTR_MAX) {
+	for_each_reserved_mem_range(&it, 0x0, UINTPTR_MAX)
+	{
 		reserved_size += it.it_limit - it.it_base + 1;
 
 		size_t last_pfn = it.it_limit / VM_PAGE_SIZE;
@@ -212,7 +224,8 @@ void vm_sparse_init(void)
 	enum sector_coverage_mode mode = get_sector_coverage_mode();
 	phys_addr_t pmem_end = 0;
 
-	enum vm_page_order sector_size = find_minimum_sector_size(last_free_pfn);
+	enum vm_page_order sector_size
+		= find_minimum_sector_size(last_free_pfn);
 	if (mode == SECTOR_COVERAGE_FREE) {
 		pmem_end = last_free_pfn * VM_PAGE_SIZE;
 	} else {
@@ -224,50 +237,90 @@ void vm_sparse_init(void)
 	size_t sector_bytes = 0;
 	unsigned int nr_sectors = 0;
 	calculate_sector_size_and_count(
-			last_reserved_pfn, last_free_pfn, pmem_end / VM_PAGE_SIZE,
-			reserved_size, free_size,
-			&nr_sectors, &sector_size);
+		last_reserved_pfn,
+		last_free_pfn,
+		pmem_end / VM_PAGE_SIZE,
+		reserved_size,
+		free_size,
+		&nr_sectors,
+		&sector_size);
 	sector_bytes = vm_page_order_to_bytes(sector_size);
 
 	char sector_size_str[64];
-	data_size_to_string(sector_bytes, sector_size_str, sizeof sector_size_str);
+	data_size_to_string(
+		sector_bytes,
+		sector_size_str,
+		sizeof sector_size_str);
 
 	sector_array = kzalloc(sizeof(struct vm_sector) * nr_sectors, 0);
 	sector_array_count = nr_sectors;
 
 	for (unsigned int i = 0; i < nr_sectors; i++) {
 		sector_array[i].s_size = sector_size;
-		sector_array[i].s_first_pfn = (i * sector_bytes) >> VM_PAGE_SHIFT;
+		sector_array[i].s_first_pfn
+			= (i * sector_bytes) >> VM_PAGE_SHIFT;
 	}
 
 	size_t s, i;
 	phys_addr_to_sector_and_index(0x3f00000, &s, &i);
 
-	for_each_free_mem_range(&it, 0x0, pmem_end) {
+	for_each_free_mem_range(&it, 0x0, pmem_end)
+	{
 		if (it.it_base & VM_PAGE_MASK) {
 			it.it_base &= ~VM_PAGE_MASK;
-			it.it_base += VM_PAGE_SIZE;
 		}
 
-		for (uintptr_t i = it.it_base; i < it.it_limit; i += VM_PAGE_SIZE) {
+		if (it.it_limit & VM_PAGE_MASK) {
+			it.it_limit &= ~VM_PAGE_MASK;
+			it.it_limit += VM_PAGE_SIZE;
+		}
+
+		for (phys_addr_t i = it.it_base; i < it.it_limit;
+		     i += VM_PAGE_SIZE) {
 			struct vm_page *pg = get_or_create_page(i);
 			pg->p_flags = 0;
 		}
 	}
 
-	for_each_reserved_mem_range(&it, 0x0, pmem_end) {
+	struct boot_module bsp;
+	bsp_get_location(&bsp);
+	if (bsp.mod_base & VM_PAGE_MASK) {
+		bsp.mod_base &= ~VM_PAGE_MASK;
+	}
+
+	if (bsp.mod_size & VM_PAGE_MASK) {
+		bsp.mod_size &= ~VM_PAGE_MASK;
+		bsp.mod_size += VM_PAGE_SIZE;
+	}
+
+	/* make sure the vm_pages for the bsp image exist, so that they can be
+	 * mapped in later */
+	for (phys_addr_t i = bsp.mod_base; i < bsp.mod_base + bsp.mod_size;
+	     i += VM_PAGE_SIZE) {
+		struct vm_page *pg = get_or_create_page(i);
+		pg->p_flags = VM_PAGE_RESERVED;
+	}
+
+	for_each_reserved_mem_range(&it, 0x0, pmem_end)
+	{
 		if (it.it_base & VM_PAGE_MASK) {
 			it.it_base &= ~VM_PAGE_MASK;
-			it.it_base += VM_PAGE_SIZE;
 		}
 
-		for (uintptr_t i = it.it_base; i < it.it_limit; i += VM_PAGE_SIZE) {
+		if (it.it_limit & VM_PAGE_MASK) {
+			it.it_limit &= ~VM_PAGE_MASK;
+			it.it_limit += VM_PAGE_SIZE;
+		}
+
+		for (phys_addr_t i = it.it_base; i < it.it_limit;
+		     i += VM_PAGE_SIZE) {
 			struct vm_page *pg = vm_page_get(i);
 
 			if (!pg) {
-				/* if the page doesn't exist, it is part of a sector
-				   that only contains reserved pages. a NULL page
-				   is implicitly treated as reserved */
+				/* if the page doesn't exist, it is part of a
+				   sector that only contains reserved pages. a
+				   NULL page is implicitly treated as reserved
+				 */
 				continue;
 			}
 
@@ -275,7 +328,9 @@ void vm_sparse_init(void)
 		}
 	}
 
-	printk("vm: [sparse] initialised %zu sectors of size %s", nr_sectors, sector_size_str);
+	printk("vm: [sparse] initialised %zu sectors of size %s",
+	       nr_sectors,
+	       sector_size_str);
 }
 
 struct vm_page *vm_page_get_sparse(phys_addr_t addr)
@@ -288,7 +343,8 @@ struct vm_page *vm_page_get_sparse(phys_addr_t addr)
 
 	struct vm_sector *sector = &sector_array[sector_number];
 
-	if (!sector->s_pages || page_number >= vm_page_order_to_pages(sector->s_size)) {
+	if (!sector->s_pages
+	    || page_number >= vm_page_order_to_pages(sector->s_size)) {
 		return NULL;
 	}
 
@@ -298,5 +354,6 @@ struct vm_page *vm_page_get_sparse(phys_addr_t addr)
 size_t vm_page_get_pfn_sparse(struct vm_page *pg)
 {
 	struct vm_sector *sector = &sector_array[pg->p_sector];
-	return sector->s_first_pfn + (((uintptr_t)pg - (uintptr_t)sector->s_pages) / sizeof *pg);
+	return sector->s_first_pfn
+	     + (((uintptr_t)pg - (uintptr_t)sector->s_pages) / sizeof *pg);
 }

vm/vm-object.c

@@ -0,0 +1,213 @@
+#include <mango/printk.h>
+#include <mango/vm-object.h>
+
+#define VM_OBJECT_CAST(p)                                                      \
+	OBJECT_C_CAST(struct vm_object, vo_base, &vm_object_type, p)
+
+static struct object_type vm_object_type = {
+	.ob_name = "vm-object",
+	.ob_size = sizeof(struct vm_object),
+	.ob_header_offset = offsetof(struct vm_object, vo_base),
+};
+
+static const enum vm_page_order GLOBAL_PAGE_ORDER = VM_PAGE_4K;
+
+static void put_page(
+	struct vm_object *vmo,
+	struct vm_page *new_page,
+	off_t offset)
+{
+	struct btree_node *cur = vmo->vo_pages.b_root;
+	new_page->p_vmo_offset = offset;
+
+	if (!cur) {
+		vmo->vo_pages.b_root = &new_page->p_bnode;
+		btree_insert_fixup(&vmo->vo_pages, &new_page->p_bnode);
+		return;
+	}
+
+	while (cur) {
+		struct vm_page *cur_page
+			= BTREE_CONTAINER(struct vm_page, p_bnode, cur);
+		struct btree_node *next = NULL;
+
+		off_t base = cur_page->p_vmo_offset;
+		off_t limit = base + vm_page_get_size_bytes(cur_page);
+		if (offset < base) {
+			next = btree_left(cur);
+		} else if (offset >= limit) {
+			next = btree_right(cur);
+		} else {
+			return;
+		}
+
+		if (next) {
+			cur = next;
+			continue;
+		}
+
+		if (offset < base) {
+			btree_put_left(cur, &new_page->p_bnode);
+		} else {
+			btree_put_right(cur, &new_page->p_bnode);
+		}
+
+		btree_insert_fixup(&vmo->vo_pages, &new_page->p_bnode);
+		return;
+	}
+}
+
+kern_status_t vm_object_type_init(void)
+{
+	return object_type_register(&vm_object_type);
+}
+
+enum vm_page_order vm_object_global_page_order(void)
+{
+	return GLOBAL_PAGE_ORDER;
+}
+
+struct vm_object *vm_object_create(
+	const char *name,
+	size_t len,
+	enum vm_prot prot)
+{
+	size_t page_bytes = VM_PAGE_SIZE;
+	uintptr_t page_mask = page_bytes - 1;
+
+	if (len & page_mask) {
+		len &= ~page_mask;
+		len += page_bytes;
+	}
+
+	struct object *obj = object_create(&vm_object_type);
+	if (!obj) {
+		return NULL;
+	}
+
+	struct vm_object *out = VM_OBJECT_CAST(obj);
+
+	out->vo_size = len;
+	out->vo_prot = prot;
+
+	if (name) {
+		strncpy(out->vo_name, name, sizeof out->vo_name);
+		out->vo_name[sizeof out->vo_name - 1] = '\0';
+	}
+
+	return out;
+}
+
+extern struct vm_object *vm_object_create_in_place(
+	const char *name,
+	phys_addr_t base,
+	size_t len,
+	enum vm_prot prot)
+{
+	struct vm_object *vmo = vm_object_create(name, len, prot);
+	if (!vmo) {
+		return NULL;
+	}
+
+	for (phys_addr_t i = base, offset = 0; i < base + vmo->vo_size;
+	     i += VM_PAGE_SIZE, offset += VM_PAGE_SIZE) {
+		struct vm_page *pg = vm_page_get(i);
+		if (!pg) {
+			printk("vm-object: invalid physical address %08llx", i);
+			object_unref(&vmo->vo_base);
+			return NULL;
+		}
+
+		put_page(vmo, pg, offset);
+	}
+
+	vmo->vo_flags |= VMO_IN_PLACE;
+
+	return vmo;
+}
+
+extern struct vm_page *vm_object_get_page(
+	const struct vm_object *vo,
+	off_t offset)
+{
+	struct btree_node *cur = vo->vo_pages.b_root;
+	while (cur) {
+		struct vm_page *page
+			= BTREE_CONTAINER(struct vm_page, p_bnode, cur);
+		struct btree_node *next = NULL;
+
+		off_t base = page->p_vmo_offset;
+		off_t limit = base + vm_page_get_size_bytes(page);
+		if (offset < base) {
+			next = btree_left(cur);
+		} else if (offset >= limit) {
+			next = btree_right(cur);
+		} else {
+			return page;
+		}
+
+		cur = next;
+	}
+
+	return NULL;
+}
+
+extern struct vm_page *vm_object_alloc_page(
+	struct vm_object *vo,
+	off_t offset,
+	enum vm_page_order size)
+{
+	struct vm_page *page = NULL;
+	struct btree_node *cur = vo->vo_pages.b_root;
+	if (!cur) {
+		page = vm_page_alloc(VM_PAGE_4K, VM_NORMAL);
+		if (!page) {
+			return NULL;
+		}
+
+		page->p_vmo_offset = offset;
+		vo->vo_pages.b_root = &page->p_bnode;
+		btree_insert_fixup(&vo->vo_pages, &page->p_bnode);
+		return page;
+	}
+
+	while (cur) {
+		struct vm_page *page
+			= BTREE_CONTAINER(struct vm_page, p_bnode, cur);
+		struct btree_node *next = NULL;
+
+		off_t base = page->p_vmo_offset;
+		off_t limit = base + vm_page_get_size_bytes(page);
+		if (offset < base) {
+			next = btree_left(cur);
+		} else if (offset >= limit) {
+			next = btree_right(cur);
+		} else {
+			return page;
+		}
+
+		if (next) {
+			cur = next;
+			continue;
+		}
+
+		page = vm_page_alloc(VM_PAGE_4K, VM_NORMAL);
+		if (!page) {
+			return NULL;
+		}
+
+		page->p_vmo_offset = offset;
+
+		if (offset < base) {
+			btree_put_left(cur, &page->p_bnode);
+		} else {
+			btree_put_right(cur, &page->p_bnode);
+		}
+
+		btree_insert_fixup(&vo->vo_pages, &page->p_bnode);
+
+		return page;
+	}
+
+	return NULL;
+}

vm/vm-region.c

@@ -0,0 +1,703 @@
+#include <mango/libc/stdio.h>
+#include <mango/object.h>
+#include <mango/panic.h>
+#include <mango/printk.h>
+#include <mango/status.h>
+#include <mango/util.h>
+#include <mango/vm-object.h>
+#include <mango/vm-region.h>
+
+#undef ASLR
+
+enum search_direction {
+	SEARCH_LEFT,
+	SEARCH_RIGHT,
+};
+
+#define VM_REGION_CAST(p)                                                      \
+	OBJECT_C_CAST(struct vm_region, vr_base, &vm_region_type, p)
+
+static struct object_type vm_region_type = {
+	.ob_name = "vm-region",
+	.ob_size = sizeof(struct vm_region),
+	.ob_header_offset = offsetof(struct vm_region, vr_base),
+};
+
+static struct vm_cache mapping_cache = {
+	.c_name = "vm-region-mapping",
+	.c_obj_size = sizeof(struct vm_region_mapping),
+};
+
+struct entry_pair {
+	struct vm_region_entry *p_left, *p_right;
+};
+
+kern_status_t vm_region_type_init(void)
+{
+	vm_cache_init(&mapping_cache);
+	return object_type_register(&vm_region_type);
+}
+
+static virt_addr_t find_free_area_linear(
+	struct vm_region *region,
+	size_t target_length);
+static virt_addr_t find_free_area_random(
+	struct vm_region *region,
+	size_t target_length);
+
+static void put_entry(struct vm_region *parent, struct vm_region_entry *child)
+{
+	struct btree_node *cur = parent->vr_entries.b_root;
+	if (!cur) {
+		parent->vr_entries.b_root = &child->e_node;
+		btree_insert_fixup(&parent->vr_entries, &child->e_node);
+		return;
+	}
+
+	virt_addr_t child_base = child->e_base_address;
+	virt_addr_t child_limit = child_base + child->e_size - 1;
+
+	while (cur) {
+		struct vm_region_entry *cur_entry
+			= BTREE_CONTAINER(struct vm_region_entry, e_node, cur);
+
+		struct btree_node *next = NULL;
+		virt_addr_t cur_base = cur_entry->e_base_address;
+		virt_addr_t cur_limit = cur_base + cur_entry->e_size - 1;
+
+		if (child_limit < cur_base) {
+			next = btree_left(cur);
+		} else if (child_base > cur_limit) {
+			next = btree_right(cur);
+		} else {
+			panic("tried to add an overlapping entry to vm-region");
+		}
+
+		if (next) {
+			cur = next;
+			continue;
+		}
+
+		if (child_limit < cur_base) {
+			btree_put_left(cur, &child->e_node);
+		} else {
+			btree_put_right(cur, &child->e_node);
+		}
+
+		btree_insert_fixup(&parent->vr_entries, &child->e_node);
+		break;
+	}
+}
+
+static struct vm_region *vm_region_from_entry(struct vm_region_entry *entry)
+{
+	if (entry->e_type != VM_REGION_ENTRY_REGION) {
+		return NULL;
+	}
+
+	return BTREE_CONTAINER(struct vm_region, vr_entry, entry);
+}
+
+static struct vm_region_mapping *vm_region_mapping_from_entry(
+	struct vm_region_entry *entry)
+{
+	if (entry->e_type != VM_REGION_ENTRY_MAPPING) {
+		return NULL;
+	}
+
+	return BTREE_CONTAINER(struct vm_region_mapping, m_entry, entry);
+}
+
+kern_status_t vm_region_create(
+	struct vm_region *parent,
+	const char *name,
+	virt_addr_t base,
+	size_t len,
+	enum vm_prot prot,
+	struct vm_region **out)
+{
+	if (!base || !len) {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	if (len & VM_PAGE_MASK) {
+		len &= ~VM_PAGE_MASK;
+		len += VM_PAGE_SIZE;
+	}
+
+	if (parent) {
+		if ((prot & parent->vr_prot) != prot) {
+			/* child region protection must match or be a
+			 * subset of parent region protection */
+			return KERN_INVALID_ARGUMENT;
+		}
+
+		if (base == VM_REGION_ANY_MAP_ADDRESS) {
+#ifdef ASLR
+			map_address = find_free_area_random(region, length);
+#else
+			base = find_free_area_linear(parent, len);
+#endif
+			base &= ~VM_PAGE_MASK;
+
+			if (base == 0) {
+				return KERN_NO_MEMORY;
+			}
+		} else if (!vm_region_is_area_free(parent, base, len)) {
+			return KERN_INVALID_ARGUMENT;
+		}
+	}
+
+	struct object *region_object = object_create(&vm_region_type);
+	if (!region_object) {
+		return KERN_NO_MEMORY;
+	}
+
+	struct vm_region *region = VM_REGION_CAST(region_object);
+
+	region->vr_prot = prot;
+	region->vr_entry.e_type = VM_REGION_ENTRY_REGION;
+	region->vr_entry.e_base_address = base;
+	region->vr_entry.e_size = len;
+
+	if (parent) {
+		region->vr_entry.e_parent = &parent->vr_entry;
+		region->vr_pmap = parent->vr_pmap;
+		put_entry(parent, &region->vr_entry);
+	}
+
+	if (name) {
+		strncpy(region->vr_name, name, sizeof region->vr_name);
+		region->vr_name[sizeof region->vr_name - 1] = '\0';
+	}
+
+	*out = region;
+	return KERN_OK;
+}
+
+static struct vm_region_entry *vm_region_find_entry(
+	struct vm_region *region,
+	virt_addr_t addr)
+{
+	struct btree_node *cur = region->vr_entries.b_root;
+	if (!cur) {
+		return NULL;
+	}
+
+	struct vm_region_entry *result = NULL;
+
+	while (cur) {
+		struct vm_region_entry *child
+			= BTREE_CONTAINER(struct vm_region_entry, e_node, cur);
+
+		struct btree_node *next = NULL;
+		virt_addr_t child_limit
+			= child->e_base_address + child->e_size - 1;
+
+		if (addr < child->e_base_address) {
+			next = btree_left(cur);
+		} else if (addr > child_limit) {
+			next = btree_right(cur);
+		} else {
+			result = child;
+			break;
+		}
+
+		cur = next;
+	}
+
+	return result;
+}
+
+struct vm_region *vm_region_find_child(
+	struct vm_region *region,
+	virt_addr_t addr)
+{
+	struct vm_region_entry *result = vm_region_find_entry(region, addr);
+
+	if (!result || result->e_type != VM_REGION_ENTRY_REGION) {
+		return region;
+	}
+
+	return vm_region_from_entry(result);
+}
+
+struct vm_region *vm_region_find_child_for_area(
+	struct vm_region *region,
+	virt_addr_t base,
+	size_t len)
+{
+	virt_addr_t limit = base + len - 1;
+
+	while (region) {
+		struct btree_node *cur = region->vr_entries.b_root;
+		if (!cur) {
+			break;
+		}
+
+		bool found_new_region = false;
+		while (cur) {
+			struct vm_region_entry *child = BTREE_CONTAINER(
+				struct vm_region_entry,
+				e_node,
+				cur);
+
+			struct btree_node *next = NULL;
+			virt_addr_t child_base = child->e_base_address;
+			virt_addr_t child_limit
+				= child_base + child->e_size - 1;
+
+			if (limit < child_base) {
+				next = btree_left(cur);
+			} else if (base > child_limit) {
+				next = btree_right(cur);
+			} else if (base >= child_base && limit <= child_limit) {
+				region = vm_region_from_entry(child);
+				found_new_region = true;
+				break;
+			} else {
+				return NULL;
+			}
+
+			cur = next;
+		}
+
+		if (!found_new_region) {
+			break;
+		}
+	}
+
+	return region;
+}
+
+struct vm_region_mapping *vm_region_find_mapping(
+	struct vm_region *region,
+	virt_addr_t addr)
+{
+	struct vm_region_entry *result = vm_region_find_entry(region, addr);
+
+	if (!result) {
+		return NULL;
+	}
+
+	return vm_region_mapping_from_entry(result);
+}
+
+static struct vm_region_entry *get_random_child(struct vm_region *region)
+{
+	enum {
+		STEP_LEFT = 0,
+		STEP_RIGHT = 1,
+		STEP_FINISH = 2,
+	} step;
+
+	struct btree_node *result = NULL;
+	struct btree_node *cur = region->vr_entries.b_root;
+	if (!cur) {
+		return NULL;
+	}
+
+	while (1) {
+		unsigned long r;
+		fill_random(&r, sizeof r);
+
+		struct btree_node *next = NULL;
+
+		step = r % 3;
+		switch (step) {
+		case STEP_LEFT:
+			next = btree_left(cur);
+			break;
+		case STEP_RIGHT:
+			next = btree_right(cur);
+			break;
+		case STEP_FINISH:
+			result = cur;
+			break;
+		default:
+			return NULL;
+		}
+
+		if (!next) {
+			result = cur;
+			break;
+		}
+
+		cur = next;
+	}
+
+	if (!result) {
+		return NULL;
+	}
+
+	return BTREE_CONTAINER(struct vm_region_entry, e_node, result);
+}
+
+static virt_addr_t find_free_area_linear_ex(
+	struct vm_region *region,
+	size_t target_length,
+	struct btree_node *start,
+	enum search_direction direction)
+{
+	if (region->vr_entry.e_size < target_length) {
+		return 0;
+	}
+
+	struct btree_node *left_node = NULL, *right_node = NULL;
+
+	switch (direction) {
+	case SEARCH_LEFT:
+		right_node = start;
+		left_node = start ? btree_left(start) : NULL;
+		break;
+	case SEARCH_RIGHT:
+		left_node = start;
+		right_node = start ? btree_left(start) : NULL;
+		break;
+	default:
+		return 0;
+	}
+
+	if (!left_node && !right_node) {
+		return 0;
+	}
+
+	while (1) {
+		struct vm_region_entry *left = BTREE_CONTAINER(
+			struct vm_region_entry,
+			e_node,
+			left_node);
+		struct vm_region_entry *right = BTREE_CONTAINER(
+			struct vm_region_entry,
+			e_node,
+			right_node);
+
+		/* addresses of the first and last free bytes in the area
+		 * respectively. */
+		virt_addr_t area_base, area_limit;
+		if (left && right) {
+			area_base = left->e_base_address + left->e_size;
+			area_limit = right->e_base_address - 1;
+		} else if (right) {
+			area_base = region->vr_entry.e_base_address;
+			area_limit = left->e_base_address - 1;
+		} else if (left) {
+			area_base = left->e_base_address + left->e_size;
+			area_limit = region->vr_entry.e_base_address
+			           + region->vr_entry.e_size - 1;
+		} else {
+			return 0;
+		}
+
+		size_t area_size = 0;
+		if (area_limit >= area_base) {
+			area_size = area_limit - area_base + 1;
+		}
+
+		if (area_size >= target_length) {
+			return area_base;
+		}
+
+		if (direction == SEARCH_RIGHT) {
+			left_node = right_node;
+			right_node = btree_next(right_node);
+		} else {
+			right_node = left_node;
+			left_node = btree_prev(right_node);
+		}
+	}
+
+	return 0;
+}
+
+static virt_addr_t find_free_area_linear(
+	struct vm_region *region,
+	size_t target_length)
+{
+	if (!region->vr_entries.b_root) {
+		return region->vr_entry.e_base_address;
+	}
+
+	return find_free_area_linear_ex(
+		region,
+		target_length,
+		btree_first(&region->vr_entries),
+		SEARCH_RIGHT);
+}
+
+static virt_addr_t random_address(
+	virt_addr_t area_base,
+	size_t area_length,
+	size_t target_length)
+{
+	size_t random_range = area_length - target_length;
+
+	off_t offset = 0;
+	fill_random(&offset, sizeof offset);
+
+	offset %= random_range;
+	return area_base + offset;
+}
+
+static virt_addr_t find_free_area_random(
+	struct vm_region *region,
+	size_t target_length)
+{
+	int tmp = 0;
+	struct btree_node *node = NULL;
+	struct vm_region_entry *basis = get_random_child(region);
+
+	fill_random(&tmp, sizeof tmp);
+	enum search_direction direction = tmp % 2;
+
+	struct vm_region_entry *left = NULL, *right = NULL;
+	if (direction == SEARCH_LEFT) {
+		node = basis ? btree_left(&basis->e_node) : NULL;
+		right = basis;
+		left = BTREE_CONTAINER(struct vm_region_entry, e_node, node);
+	} else {
+		node = basis ? btree_right(&basis->e_node) : NULL;
+		left = basis;
+		right = BTREE_CONTAINER(struct vm_region_entry, e_node, node);
+	}
+
+	virt_addr_t base = region->vr_entry.e_base_address,
+		    limit = base + region->vr_entry.e_size - 1;
+
+	if (left) {
+		base = left->e_base_address;
+	}
+
+	if (right) {
+		limit = right->e_base_address + right->e_size - 1;
+	}
+
+	return random_address(base, limit - base + 1, target_length);
+}
+
+kern_status_t vm_region_map_object(
+	struct vm_region *region,
+	virt_addr_t map_address,
+	struct vm_object *object,
+	off_t object_offset,
+	size_t length,
+	enum vm_prot prot,
+	virt_addr_t *out)
+{
+
+	object_offset &= ~VM_PAGE_MASK;
+
+	if (length & VM_PAGE_MASK) {
+		length &= ~VM_PAGE_MASK;
+		length += VM_PAGE_SIZE;
+	}
+
+	if (!region || !object || !out) {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	if ((prot & region->vr_prot) != prot) {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	if ((prot & object->vo_prot) != prot) {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	if (!length || object_offset + length > object->vo_size) {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	if (map_address != VM_REGION_ANY_MAP_ADDRESS) {
+		region = vm_region_find_child_for_area(
+			region,
+			map_address,
+			length);
+	}
+
+	if (!region) {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	if (map_address == VM_REGION_ANY_MAP_ADDRESS) {
+#ifdef ASLR
+		map_address = find_free_area_random(region, length);
+#else
+		map_address = find_free_area_linear(region, length);
+#endif
+		map_address &= ~VM_PAGE_MASK;
+
+		if (map_address == 0) {
+			return KERN_NO_MEMORY;
+		}
+	} else if (!vm_region_is_area_free(region, map_address, length)) {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	struct vm_region_mapping *mapping
+		= vm_cache_alloc(&mapping_cache, VM_NORMAL);
+	if (!mapping) {
+		return KERN_NO_MEMORY;
+	}
+
+	tracek("mapping %s at [%llx-%llx]",
+	       object->vo_name,
+	       map_address,
+	       map_address + length);
+	mapping->m_object = object;
+	mapping->m_prot = prot;
+	mapping->m_object_offset = object_offset;
+	mapping->m_entry.e_type = VM_REGION_ENTRY_MAPPING;
+	mapping->m_entry.e_parent = &region->vr_entry;
+	mapping->m_entry.e_base_address = map_address;
+	mapping->m_entry.e_size = length;
+
+	put_entry(region, &mapping->m_entry);
+	queue_push_back(&object->vo_mappings, &mapping->m_object_entry);
+
+	*out = map_address;
+	return KERN_OK;
+}
+
+bool vm_region_is_area_free(
+	const struct vm_region *region,
+	virt_addr_t base,
+	size_t len)
+{
+	/* address of the last byte in the region */
+	virt_addr_t region_limit
+		= region->vr_entry.e_base_address + region->vr_entry.e_size - 1;
+	if (base < region->vr_entry.e_base_address || base > region_limit) {
+		return false;
+	}
+
+	if (base + len - 1 > region_limit) {
+		return false;
+	}
+
+	virt_addr_t limit = base + len - 1;
+
+	struct btree_node *cur = region->vr_entries.b_root;
+	if (!cur) {
+		return true;
+	}
+
+	while (cur) {
+		struct vm_region_entry *entry
+			= BTREE_CONTAINER(struct vm_region_entry, e_node, cur);
+
+		struct btree_node *next = NULL;
+		virt_addr_t entry_limit
+			= entry->e_base_address + entry->e_size - 1;
+
+		if (base > entry_limit) {
+			next = btree_right(cur);
+		} else if (limit < entry->e_base_address) {
+			next = btree_left(cur);
+		} else {
+			return false;
+		}
+
+		cur = next;
+	}
+
+	return true;
+}
+
+kern_status_t vm_region_demand_map(
+	struct vm_region *region,
+	virt_addr_t addr,
+	enum pmap_fault_flags flags)
+{
+	addr &= ~VM_PAGE_MASK;
+	region = vm_region_find_child(region, addr);
+
+	struct vm_region_mapping *mapping
+		= vm_region_find_mapping(region, addr);
+	if (!mapping) {
+		return KERN_NO_ENTRY;
+	}
+
+	off_t offset = addr - mapping->m_entry.e_base_address
+	             + mapping->m_object_offset;
+
+	tracek("vm: tried to access vm-object %s at offset=%05llx",
+	       mapping->m_object->vo_name,
+	       offset);
+
+	struct vm_page *pg
+		= vm_object_alloc_page(mapping->m_object, offset, VM_PAGE_4K);
+	tracek("vm: mapping %07llx -> %10llx", vm_page_get_paddr(pg), addr);
+	return pmap_add(
+		region->vr_pmap,
+		addr,
+		vm_page_get_pfn(pg),
+		mapping->m_prot,
+		PMAP_NORMAL);
+}
+
+#ifdef TRACE
+void vm_region_dump(struct vm_region *region, int depth)
+{
+	char line[128] = {0};
+	size_t p = 0;
+
+	for (int i = 0; i < depth; i++) {
+		p += snprintf(line + p, sizeof line - p, "   ");
+	}
+	p += snprintf(
+		line + p,
+		sizeof line - p,
+		"region: %s [%llx-%llx]",
+		region->vr_name,
+		region->vr_entry.e_base_address,
+		region->vr_entry.e_base_address + region->vr_entry.e_size);
+
+	printk("%s", line);
+
+	struct btree_node *cur = btree_first(&region->vr_entries);
+	while (cur) {
+		memset(line, 0x0, sizeof line);
+		p = 0;
+
+		for (int i = 0; i < depth + 1; i++) {
+			p += snprintf(line + p, sizeof line - p, "   ");
+		}
+
+		struct vm_region_entry *entry
+			= BTREE_CONTAINER(struct vm_region_entry, e_node, cur);
+		struct vm_region *child_region = vm_region_from_entry(entry);
+		struct vm_region_mapping *child_mapping
+			= vm_region_mapping_from_entry(entry);
+
+		switch (entry->e_type) {
+		case VM_REGION_ENTRY_REGION:
+			break;
+		case VM_REGION_ENTRY_MAPPING:
+			p += snprintf(
+				line + p,
+				sizeof line - p,
+				"mapping: %s [%llx-%llx] -> [%llx-%llx]",
+				child_mapping->m_object->vo_name,
+				child_mapping->m_object_offset,
+				child_mapping->m_object_offset
+					+ child_mapping->m_entry.e_size,
+				child_mapping->m_entry.e_base_address,
+				child_mapping->m_entry.e_base_address
+					+ child_mapping->m_entry.e_size);
+			printk("%s", line);
+			break;
+		default:
+			p += snprintf(line + p, sizeof line - p, "invalid");
+			printk("%s", line);
+			break;
+		}
+
+		if (child_region) {
+			vm_region_dump(child_region, depth + 1);
+		}
+
+		cur = btree_next(cur);
+	}
+}
+#endif