libmango: types: add macro to define a kern_msg_handle_t

when locking a pair of objects, the object with the lesser memory address
is always locked first. the pair is unlocked in the opposite order.

CMakeLists.txt

@@ -12,12 +12,11 @@ set(kernel_arch x86_64)
 set(kernel_name "Mango")
 set(kernel_exe_name "mango_kernel")
 
-set(generic_src_dirs ds init kernel libc sched util vm)
+set(generic_src_dirs ds init kernel libc sched util vm syscall)
 set(kernel_sources "")
 set(kernel_headers "")
 
 foreach (dir ${generic_src_dirs})
-	message(STATUS ${dir})
 	file(GLOB_RECURSE dir_sources ${dir}/*.c)
 	file(GLOB_RECURSE dir_headers ${dir}/*.h)
 
@@ -41,6 +40,7 @@ add_executable(${kernel_exe_name}
 target_include_directories(${kernel_exe_name} PRIVATE
 	include
 	libc/include
+	libmango/include
 	arch/${kernel_arch}/include)
 target_compile_options(${kernel_exe_name} PRIVATE
 	-nostdlib -ffreestanding

arch/user/cpu.c

@@ -1,5 +1,5 @@
 #include <unistd.h>
-#include <mango/machine/cpu.h>
+#include <kernel/machine/cpu.h>
 
 int ml_init_bootcpu(void)
 {

arch/user/hwlock.c

@@ -1,5 +1,5 @@
-#include <mango/machine/hwlock.h>
+#include <kernel/compiler.h>
-#include <mango/compiler.h>
+#include <kernel/machine/hwlock.h>
 
 void ml_hwlock_lock(ml_hwlock_t *lck)
 {

arch/user/include/mango/machine/init.h

@@ -1,6 +1,7 @@
 #ifndef MANGO_X86_64_INIT_H_
 #define MANGO_X86_64_INIT_H_
 
+#include <stddef.h>
 #include <stdint.h>
 
 #ifdef __cplusplus
@@ -8,16 +9,18 @@ extern "C" {
 #endif
 
 #define __X2(x) #x
-#define __X(x) __X2(x)
+#define __X(x)  __X2(x)
 
 #ifdef __APPLE__
-#define __define_initcall(fn, id) \
+#define __define_initcall(fn, id)                                              \
-	static initcall_t __initcall_##fn##id __used \
+	static initcall_t __initcall_##fn##id __used __section(                \
-	__section("__DATA,__initcall" __X(id) ".init") = (fn)
+		"__DATA,__initcall" __X(id) ".init")                           \
+		= (fn)
 #else
-#define __define_initcall(fn, id) \
+#define __define_initcall(fn, id)                                              \
-	static initcall_t __initcall_##fn##id __used \
+	static initcall_t __initcall_##fn##id __used __section(                \
-	__section("initcall" __X(id) "_init") = (fn)
+		"initcall" __X(id) "_init")                                    \
+		= (fn)
 #endif
 
 extern int ml_init(uintptr_t arg);

arch/user/include/mango/machine/pmap.h

@@ -1,6 +1,8 @@
 #ifndef MANGO_USER_PMAP_H_
 #define MANGO_USER_PMAP_H_
 
+#include <stdint.h>
+
 typedef uintptr_t ml_pmap_t;
 typedef uint64_t ml_pfn_t;
 

arch/user/init.c

@@ -1,11 +1,11 @@
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <mango/init.h>
+#include <kernel/init.h>
-#include <mango/memblock.h>
+#include <kernel/memblock.h>
-#include <mango/vm.h>
+#include <kernel/vm.h>
-#include <mango/object.h>
+#include <kernel/object.h>
-#include <mango/printk.h>
+#include <kernel/printk.h>
 #include <arch/stdcon.h>
 #include <sys/mman.h>
 

arch/user/initcall.c

@@ -1,4 +1,4 @@
-#include <mango/init.h>
+#include <kernel/init.h>
 
 #ifdef __APPLE__
 extern char __start_initcall0[] __asm("section$start$__DATA$__initcall0.init");

arch/user/stdcon.c

@@ -1,10 +1,10 @@
-#include <mango/libc/string.h>
+#include <kernel/libc/string.h>
-#include <mango/libc/ctype.h>
+#include <kernel/libc/ctype.h>
 #include <stdint.h>
 #include <stdio.h>
-#include <mango/console.h>
+#include <kernel/console.h>
-#include <mango/vm.h>
+#include <kernel/vm.h>
-#include <mango/printk.h>
+#include <kernel/printk.h>
 
 static void stdcon_write(struct console *con, const char *s, unsigned int len)
 {

arch/x86_64/asm-offset.c

@@ -1,5 +1,5 @@
-#include <mango/sched.h>
+#include <kernel/sched.h>
-#include <mango/compiler.h>
+#include <kernel/compiler.h>
 
 //size_t THREAD_sp = offsetof(struct thread, tr_sp);
 

arch/x86_64/config.cmake

@@ -2,9 +2,3 @@ target_compile_options(${kernel_exe_name} PRIVATE
 	-z max-page-size=0x1000 -m64 -mcmodel=large -mno-red-zone -mno-mmx
 	-mno-sse -mno-sse2 -D_64BIT -DBYTE_ORDER=1234)
 target_link_libraries(${kernel_exe_name} "-z max-page-size=0x1000" "-T ${CMAKE_CURRENT_SOURCE_DIR}/arch/x86_64/layout.ld")
-
-add_custom_command(TARGET ${kernel_exe_name} POST_BUILD
-	COMMAND ${BUILD_TOOLS_DIR}/e64patch $<TARGET_FILE:${kernel_exe_name}>
-	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-	COMMENT "Patching kernel elf64 image"
-)

arch/x86_64/cpu.c

@@ -1,11 +1,14 @@
-#include <mango/machine/cpu.h>
 #include <arch/msr.h>
+#include <kernel/machine/cpu.h>
 
 int ml_cpu_block_init(ml_cpu_block *p)
 {
 	p->c_this = p;
 	gdt_init(&p->c_gdt, &p->c_gdt_ptr);
 	idt_init(&p->c_idt_ptr);
+	tss_init(&p->c_tss, &p->c_tss_ptr);
+
+	gdt_write_tss(&p->c_gdt, &p->c_tss);
 	return 0;
 }
 
@@ -13,6 +16,27 @@ int ml_cpu_block_use(ml_cpu_block *p)
 {
 	gdt_load(&p->c_gdt_ptr);
 	idt_load(&p->c_idt_ptr);
+	tss_load(&p->c_tss);
 	wrmsr(MSR_GS_BASE, (uint64_t)p);
 	return 0;
 }
+
+virt_addr_t ml_cpu_block_get_kstack(ml_cpu_block *p)
+{
+	return tss_get_kstack(&p->c_tss);
+}
+
+virt_addr_t ml_cpu_block_get_ustack(ml_cpu_block *p)
+{
+	return tss_get_ustack(&p->c_tss);
+}
+
+void ml_cpu_block_set_kstack(ml_cpu_block *p, virt_addr_t sp)
+{
+	tss_set_kstack(&p->c_tss, sp);
+}
+
+void ml_cpu_block_set_ustack(ml_cpu_block *p, virt_addr_t sp)
+{
+	tss_set_ustack(&p->c_tss, sp);
+}

arch/x86_64/e820.c

@@ -1,8 +1,8 @@
-#include "mango/types.h"
-#include <mango/memblock.h>
-#include <mango/printk.h>
-#include <mango/util.h>
 #include <arch/e820.h>
+#include <kernel/memblock.h>
+#include <kernel/printk.h>
+#include <kernel/types.h>
+#include <kernel/util.h>
 
 void e820_scan(multiboot_memory_map_t *mmap, size_t len)
 {
@@ -36,7 +36,9 @@ void e820_scan(multiboot_memory_map_t *mmap, size_t len)
 		}
 
 		printk("e820: [mem 0x%016llx-0x%016llx] %s",
-		       entry->addr, entry->addr + entry->len - 1, type);
+		       entry->addr,
+		       entry->addr + entry->len - 1,
+		       type);
 
 		memblock_add(entry->addr, entry->len);
 
@@ -53,7 +55,12 @@ void e820_scan(multiboot_memory_map_t *mmap, size_t len)
 
 	char str_mem_total[64], str_mem_reserved[64];
 	data_size_to_string(mem_total, str_mem_total, sizeof str_mem_total);
-	data_size_to_string(mem_reserved, str_mem_reserved, sizeof str_mem_reserved);
+	data_size_to_string(
+		mem_reserved,
+		str_mem_reserved,
+		sizeof str_mem_reserved);
 
-	printk("e820: total memory: %s, hw reserved: %s", str_mem_total, str_mem_reserved);
+	printk("e820: total memory: %s, hw reserved: %s",
+	       str_mem_total,
+	       str_mem_reserved);
 }

arch/x86_64/gdt.c

@@ -1,5 +1,8 @@
-#include <mango/libc/string.h>
 #include <arch/gdt.h>
+#include <arch/tss.h>
+#include <kernel/libc/string.h>
+#include <kernel/types.h>
+#include <stddef.h>
 
 static void init_entry(struct gdt_entry *entry, int access, int flags)
 {
@@ -15,13 +18,42 @@ int gdt_init(struct gdt *gdt, struct gdt_ptr *gdtp)
 {
 	memset(&gdt->g_entries[0], 0x0, sizeof gdt->g_entries[0]);
 
-	init_entry(&gdt->g_entries[1], GDT_A_PRESENT | GDT_A_CODEREAD | GDT_A_CODE, GDT_F_64BIT);
+	init_entry(
-	init_entry(&gdt->g_entries[2], GDT_A_PRESENT | GDT_A_DATAWRITE | GDT_A_DATA, GDT_F_64BIT);
+		&gdt->g_entries[1],
-	init_entry(&gdt->g_entries[3], GDT_A_PRESENT | GDT_A_USER | GDT_A_CODEREAD | GDT_A_CODE, GDT_F_64BIT);
+		GDT_A_PRESENT | GDT_A_CODEREAD | GDT_A_CODE,
-	init_entry(&gdt->g_entries[4], GDT_A_PRESENT | GDT_A_USER | GDT_A_DATAWRITE | GDT_A_DATA, GDT_F_64BIT);
+		GDT_F_64BIT);
+	init_entry(
+		&gdt->g_entries[2],
+		GDT_A_PRESENT | GDT_A_DATAWRITE | GDT_A_DATA,
+		GDT_F_64BIT);
+	init_entry(
+		&gdt->g_entries[3],
+		GDT_A_PRESENT | GDT_A_USER | GDT_A_DATAWRITE | GDT_A_DATA,
+		GDT_F_64BIT);
+	init_entry(
+		&gdt->g_entries[4],
+		GDT_A_PRESENT | GDT_A_USER | GDT_A_CODEREAD | GDT_A_CODE,
+		GDT_F_64BIT);
 
 	gdtp->g_ptr = (uint64_t)gdt;
 	gdtp->g_limit = sizeof(*gdt) - 1;
 
 	return 0;
 }
+
+void gdt_write_tss(struct gdt *gdt, struct tss *tss)
+{
+	struct tss_gdt_entry *tss_entry = &gdt->g_tss;
+
+	virt_addr_t base = (virt_addr_t)tss;
+	size_t limit = sizeof *tss;
+
+	tss_entry->ge_base_low = (base & 0xFFFF);
+	tss_entry->ge_base_mid = (base >> 16) & 0xFF;
+	tss_entry->ge_base_hi = (base >> 24) & 0xFF;
+	tss_entry->ge_base_ext = (base >> 32) & 0xFFFFFFFF;
+	tss_entry->ge_limit_low = (limit & 0xFFFF);
+	tss_entry->ge_gran = (limit >> 16) & 0xF;
+	tss_entry->ge_access = 0xE9;
+	tss_entry->ge_reserved = 0;
+}

arch/x86_64/hwlock.S

@@ -11,6 +11,41 @@ ml_hwlock_lock:
 
 	mov $1, %ecx
 
+	mfence
+
+1:	mov $0, %eax
+	lock cmpxchg %ecx, (%rdi)
+	jne 1b
+
+	pop %rbp
+	ret
+
+
+	.global ml_hwlock_unlock
+	.type ml_hwlock_unlock, @function
+
+	/* %rdi = pointer to ml_hwlock_t (int) */
+ml_hwlock_unlock:
+	push %rbp
+	mov %rsp, %rbp
+
+	movl $0, (%rdi)
+	mfence
+
+	pop %rbp
+	ret
+
+
+	.global ml_hwlock_lock_irq
+	.type ml_hwlock_lock_irq, @function
+
+	/* %rdi = pointer to ml_hwlock_t (int) */
+ml_hwlock_lock_irq:
+	push %rbp
+	mov %rsp, %rbp
+
+	mov $1, %ecx
+
 	cli
 	mfence
 
@@ -21,11 +56,12 @@ ml_hwlock_lock:
 	pop %rbp
 	ret
 
-	.global ml_hwlock_unlock
+
-	.type ml_hwlock_unlock, @function
+	.global ml_hwlock_unlock_irq
+	.type ml_hwlock_unlock_irq, @function
 
 	/* %rdi = pointer to ml_hwlock_t (int) */
-ml_hwlock_unlock:
+ml_hwlock_unlock_irq:
 	push %rbp
 	mov %rsp, %rbp
 
@@ -42,7 +78,7 @@ ml_hwlock_unlock:
 
 	/* %rdi = pointer to ml_hwlock_t (int)
 	   %rsi = pointer to quadword to store rflags in */
-ml_hwlock_lock_irqsave:	
+ml_hwlock_lock_irqsave:
 	push %rbp
 	mov %rsp, %rbp
 
@@ -62,6 +98,7 @@ ml_hwlock_lock_irqsave:
 	pop %rbp
 	ret
 
+
 	.global ml_hwlock_unlock_irqrestore
 	.type ml_hwlock_unlock_irqrestore, @function
 

arch/x86_64/include/arch/gdt.h

@@ -1,26 +1,27 @@
 #ifndef ARCH_GDT_H_
 #define ARCH_GDT_H_
 
-#include <mango/compiler.h>
+#include <arch/tss.h>
+#include <kernel/compiler.h>
 #include <stdint.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#define NR_GDT_ENTRIES 5
+#define NR_GDT_ENTRIES  5
 
-#define GDT_A_PRESENT		(1 << 7)
+#define GDT_A_PRESENT   (1 << 7)
-#define GDT_A_USER		(3 << 5)
+#define GDT_A_USER      (3 << 5)
-#define GDT_A_CODE		(3 << 3)
+#define GDT_A_CODE      (3 << 3)
-#define GDT_A_DATA		(8 << 1)
+#define GDT_A_DATA      (8 << 1)
-#define GDT_A_RPLK		(1 << 5)
+#define GDT_A_RPLK      (1 << 5)
-#define GDT_A_CODEREAD		(1 << 1)
+#define GDT_A_CODEREAD  (1 << 1)
-#define GDT_A_DATAWRITE		(1 << 1)
+#define GDT_A_DATAWRITE (1 << 1)
-#define GDT_A_GROWUP		(1 << 5)
+#define GDT_A_GROWUP    (1 << 5)
-#define GDT_F_64BIT		(0x2F)
+#define GDT_F_64BIT     (0x2F)
-#define GDT_F_32BIT		(0x4F)
+#define GDT_F_32BIT     (0x4F)
-#define GDT_F_16BIT		(0xF)
+#define GDT_F_16BIT     (0xF)
 
 struct gdt_entry {
 	uint16_t ge_limit_low;
@@ -33,6 +34,7 @@ struct gdt_entry {
 
 struct gdt {
 	struct gdt_entry g_entries[NR_GDT_ENTRIES];
+	struct tss_gdt_entry g_tss;
 } __packed;
 
 struct gdt_ptr {
@@ -41,6 +43,7 @@ struct gdt_ptr {
 } __packed;
 
 extern int gdt_init(struct gdt *gdt, struct gdt_ptr *gdtp);
+extern void gdt_write_tss(struct gdt *gdt, struct tss *tss);
 extern int gdt_load(struct gdt_ptr *gdtp);
 
 #ifdef __cplusplus

arch/x86_64/include/arch/irq.h

@@ -1,8 +1,8 @@
 #ifndef ARCH_IRQ_H_
 #define ARCH_IRQ_H_
 
-#include <mango/compiler.h>
+#include <kernel/compiler.h>
-#include <mango/queue.h>
+#include <kernel/queue.h>
 #include <stdint.h>
 
 #ifdef __cplusplus
@@ -11,6 +11,8 @@ extern "C" {
 
 #define NR_IDT_ENTRIES 256
 
+struct ml_cpu_context;
+
 enum irq_vector {
 	IRQ0 = 32,
 	IRQ1,
@@ -35,13 +37,6 @@ struct irq_hook {
 	int (*irq_callback)(void);
 };
 
-struct cpu_context {
-	uint64_t r15, r14, r13, r12, r11, r10, r9, r8;
-	uint64_t rdi, rsi, rbp, unused_rsp, rbx, rdx, rcx, rax;
-	uint64_t int_no, err_no;
-	uint64_t rip, cs, rflags, rsp, ss;
-} __packed;
-
 struct idt_entry {
 	uint16_t base_low;
 	uint16_t selector;
@@ -64,7 +59,7 @@ struct idt_ptr {
 	uintptr_t i_base;
 } __packed;
 
-typedef void (*int_hook)(struct cpu_context *);
+typedef void (*int_hook)(struct ml_cpu_context *);
 
 extern int idt_init(struct idt_ptr *idtp);
 extern int idt_load(struct idt_ptr *idtp);

arch/x86_64/include/arch/msr.h

@@ -3,7 +3,8 @@
 
 #include <stdint.h>
 
-#define MSR_GS_BASE 0xC0000101
+#define MSR_GS_BASE        0xC0000101
+#define MSR_KERNEL_GS_BASE 0xC0000102
 
 #ifdef __cplusplus
 extern "C" {

arch/x86_64/include/arch/paging.h

@@ -1,8 +1,8 @@
 #ifndef ARCH_PAGING_H_
 #define ARCH_PAGING_H_
 
-#include <mango/types.h>
+#include <kernel/types.h>
-#include <mango/compiler.h>
+#include <kernel/compiler.h>
 
 #ifdef __cplusplus
 extern "C" {

arch/x86_64/include/arch/tss.h

@@ -0,0 +1,55 @@
+#ifndef ARCH_TSS_H_
+#define ARCH_TSS_H_
+
+#include <kernel/compiler.h>
+#include <kernel/types.h>
+#include <stdint.h>
+
+#define TSS_GDT_INDEX 5
+
+struct tss {
+	uint32_t res0;
+	uint64_t rsp0;
+	uint64_t rsp1;
+	uint64_t rsp2;
+	uint64_t res1;
+	uint64_t ist1;
+	uint64_t ist2;
+	uint64_t ist3;
+	uint64_t ist4;
+	uint64_t ist5;
+	uint64_t ist6;
+	uint64_t ist7;
+	uint64_t res2;
+	uint16_t res3;
+	uint16_t iomap_offset;
+} __packed;
+
+struct tss_gdt_entry {
+	/* these fields are copied from struct gdt_entry */
+	uint16_t ge_limit_low;
+	uint16_t ge_base_low;
+	uint8_t ge_base_mid;
+	uint8_t ge_access;
+	uint8_t ge_gran;
+	uint8_t ge_base_hi;
+	/* these fields are specific to the TSS entry */
+	uint32_t ge_base_ext;
+	uint32_t ge_reserved;
+} __packed;
+
+struct tss_ptr {
+	uint16_t tss_limit;
+	uint64_t tss_base;
+} __packed;
+
+extern void tss_init(struct tss *tss, struct tss_ptr *ptr);
+extern void tss_load(struct tss *tss);
+
+extern virt_addr_t tss_get_kstack(struct tss *tss);
+extern virt_addr_t tss_get_ustack(struct tss *tss);
+
+extern void tss_set_kstack(struct tss *tss, virt_addr_t sp);
+extern void tss_set_ustack(struct tss *tss, virt_addr_t sp);
+
+#endif

arch/x86_64/include/kernel/machine/cpu.h

@@ -0,0 +1,77 @@
+#ifndef KERNEL_X86_64_CPU_H_
+#define KERNEL_X86_64_CPU_H_
+
+#include <arch/gdt.h>
+#include <arch/irq.h>
+#include <arch/tss.h>
+#include <kernel/types.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define ML_BIG_ENDIAN            0
+
+#define ml_cpu_block_get_id(p)   ((p)->c_cpu_id)
+#define ml_cpu_block_get_data(p) ((p)->c_data)
+
+#if 0
+#define ml_read_sp(sp, bp)                                                     \
+	asm volatile("mov %%rsp, %0" : "=r"(sp));                              \
+	asm volatile("mov %%rbp, %0" : "=r"(bp));
+#endif
+
+struct cpu_data;
+
+typedef struct ml_cpu_block {
+	struct ml_cpu_block *c_this;
+
+	struct gdt c_gdt;
+	struct gdt_ptr c_gdt_ptr;
+
+	struct tss c_tss;
+	struct tss_ptr c_tss_ptr;
+
+	struct idt_ptr c_idt_ptr;
+	unsigned int c_cpu_id;
+
+	struct cpu_data *c_data;
+} ml_cpu_block;
+
+struct ml_int_context {
+	uint64_t rip, cs, rflags, rsp, ss;
+};
+
+struct ml_cpu_context {
+	uint64_t r15, r14, r13, r12, r11, r10, r9, r8;
+	uint64_t rdi, rsi, rbp, unused_rsp, rbx, rdx, rcx, rax;
+	uint64_t int_no, err_no;
+	uint64_t rip, cs, rflags, rsp, ss;
+} __packed;
+
+#define ml_cpu_pause()   __asm__ __volatile__("hlt")
+#define ml_cpu_relax()   __asm__ __volatile__("pause")
+
+#define ml_int_disable() __asm__ __volatile__("cli")
+#define ml_int_enable()  __asm__ __volatile__("sti")
+
+extern int ml_init_bootcpu(void);
+
+extern int ml_cpu_block_init(ml_cpu_block *p);
+extern int ml_cpu_block_use(ml_cpu_block *p);
+
+extern virt_addr_t ml_cpu_block_get_ustack(ml_cpu_block *p);
+extern virt_addr_t ml_cpu_block_get_kstack(ml_cpu_block *p);
+
+extern void ml_cpu_block_set_ustack(ml_cpu_block *p, virt_addr_t sp);
+extern void ml_cpu_block_set_kstack(ml_cpu_block *p, virt_addr_t sp);
+
+/* defined in cpu_ctrl.S */
+extern void ml_halt_cpu(void);
+extern ml_cpu_block *ml_this_cpu(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

arch/x86_64/include/kernel/machine/hwlock.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_X86_64_HWLOCK_H_
+#ifndef KERNEL_X86_64_HWLOCK_H_
-#define MANGO_X86_64_HWLOCK_H_
+#define KERNEL_X86_64_HWLOCK_H_
 
 #define ML_HWLOCK_INIT (0)
 
@@ -12,6 +12,9 @@ typedef int ml_hwlock_t;
 extern void ml_hwlock_lock(ml_hwlock_t *lck);
 extern void ml_hwlock_unlock(ml_hwlock_t *lck);
 
+extern void ml_hwlock_lock_irq(ml_hwlock_t *lck);
+extern void ml_hwlock_unlock_irq(ml_hwlock_t *lck);
+
 extern void ml_hwlock_lock_irqsave(ml_hwlock_t *lck, unsigned long *flags);
 extern void ml_hwlock_unlock_irqrestore(ml_hwlock_t *lck, unsigned long flags);
 

arch/x86_64/include/kernel/machine/init.h

@@ -0,0 +1,28 @@
+#ifndef KERNEL_X86_64_INIT_H_
+#define KERNEL_X86_64_INIT_H_
+
+#include <stddef.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define __X2(x) #x
+#define __X(x)  __X2(x)
+
+#define __define_initcall(fn, id)                                              \
+	static initcall_t __initcall_##fn##id __used __section(                \
+		".initcall" __X(id) ".init")                                   \
+		= (fn)
+
+extern int ml_init(uintptr_t arg);
+
+extern const struct framebuffer_varinfo *bootfb_varinfo(void);
+extern const struct framebuffer_fixedinfo *bootfb_fixedinfo(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

arch/x86_64/include/kernel/machine/irq.h

@@ -0,0 +1,5 @@
+#ifndef KERNEL_X86_64_IRQ_H_
+#define KERNEL_X86_64_IRQ_H_
+
+
+#endif

arch/x86_64/include/kernel/machine/panic.h

@@ -0,0 +1,12 @@
+#ifndef KERNEL_X86_64_PANIC_H_
+#define KERNEL_X86_64_PANIC_H_
+
+#include <stdint.h>
+
+struct ml_cpu_context;
+
+extern void ml_print_cpu_state(struct ml_cpu_context *ctx);
+extern void ml_print_stack_trace(uintptr_t ip);
+extern void ml_print_stack_trace_irq(struct ml_cpu_context *ctx);
+
+#endif

arch/x86_64/include/kernel/machine/pmap.h

@@ -0,0 +1,11 @@
+#ifndef KERNEL_X86_64_PMAP_H_
+#define KERNEL_X86_64_PMAP_H_
+
+#include <arch/paging.h>
+
+#define ML_PMAP_INVALID ((uintptr_t)-1)
+
+typedef pml4t_ptr_t ml_pmap_t;
+typedef uint64_t ml_pfn_t;
+
+#endif

arch/x86_64/include/kernel/machine/thread.h

@@ -0,0 +1,31 @@
+#ifndef KERNEL_X86_64_THREAD_H_
+#define KERNEL_X86_64_THREAD_H_
+
+#include <kernel/sched.h>
+
+struct ml_cpu_context;
+
+/* switch from one thread to another. the stack of the `to` thread must have
+ * been prepared in one of two ways:
+ *  1) a previous call to ml_thread_switch where it was the `from` thread.
+ *  2) a call to ml_thread_prepare_kernel_context
+ * the switch occurs entirely with kernel mode. a further return from an
+ * interrupt context is then used to return to usermode if necessary.
+ */
+extern void ml_thread_switch(struct thread *from, struct thread *to);
+
+/* perform the initial transition to userspace. the stack must be prepared using
+ * ml_thread_prepare_user_context before this function can be used */
+extern void ml_thread_switch_user(void);
+/* prepare the stack so that ml_thread_switch can jump to the specified IP/SP */
+extern void ml_thread_prepare_kernel_context(uintptr_t ip, uintptr_t *sp);
+/* prepare the stack so that ml_thread_switch_user can jump to usermode
+ * with the specified IP/user SP */
+extern kern_status_t ml_thread_prepare_user_context(
+	virt_addr_t ip,
+	virt_addr_t user_sp,
+	virt_addr_t *kernel_sp,
+	const uintptr_t *args,
+	size_t nr_args);
+
+#endif

arch/x86_64/include/kernel/machine/vm.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_X86_64_VM_H_
+#ifndef KERNEL_X86_64_VM_H_
-#define MANGO_X86_64_VM_H_
+#define KERNEL_X86_64_VM_H_
 
 /* kernel higher-half base virtual address. */
 #define VM_KERNEL_VOFFSET 0xFFFFFFFF80000000
@@ -11,14 +11,20 @@
 #define VM_PAGEMAP_BASE   0xFFFF888000000000
 #define VM_PAGEMAP_LIMIT  0xFFFFC87FFFFFFFFF
 
-#define VM_PAGE_SIZE  0x1000
+#define VM_PAGE_SIZE      0x1000
-#define VM_PAGE_MASK  (VM_PAGE_SIZE-1)
+#define VM_PAGE_MASK      (VM_PAGE_SIZE - 1)
-#define VM_PAGE_SHIFT 12
+#define VM_PAGE_SHIFT     12
 
 #define VM_PAGE_MIN_ORDER VM_PAGE_4K
 #define VM_PAGE_MAX_ORDER VM_PAGE_8M
 
-#define VM_ZONE_MIN VM_ZONE_DMA
+#define VM_ZONE_MIN       VM_ZONE_DMA
-#define VM_ZONE_MAX VM_ZONE_NORMAL
+#define VM_ZONE_MAX       VM_ZONE_NORMAL
+
+#define VM_USER_BASE      0x0000000000100000
+#define VM_USER_LIMIT     0x00007fffffffffff
+
+#define VM_KERNEL_BASE    0XFFFF800000000000
+#define VM_KERNEL_LIMIT   0XFFFFFFFFFFFFFFFF
 
 #endif

arch/x86_64/include/mango/machine/cpu.h

@@ -1,49 +0,0 @@
-#ifndef MANGO_X86_64_CPU_H_
-#define MANGO_X86_64_CPU_H_
-
-#include <arch/gdt.h>
-#include <arch/irq.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define ML_BIG_ENDIAN 0
-
-#define ml_cpu_block_get_id(p) ((p)->c_cpu_id)
-#define ml_cpu_block_get_data(p) ((p)->c_data)
-
-struct cpu_data;
-
-typedef struct ml_cpu_block {
-	struct ml_cpu_block *c_this;
-
-	struct gdt c_gdt;
-	struct gdt_ptr c_gdt_ptr;
-
-	struct idt_ptr c_idt_ptr;
-	unsigned int c_cpu_id;
-
-	struct cpu_data *c_data;
-} ml_cpu_block;
-
-#define ml_cpu_pause() __asm__ __volatile__("hlt")
-#define ml_cpu_relax() __asm__ __volatile__("pause")
-
-#define ml_int_disable() __asm__ __volatile__("cli")
-#define ml_int_enable() __asm__ __volatile__("sti")
-
-extern int ml_init_bootcpu(void);
-
-extern int ml_cpu_block_init(ml_cpu_block *p);
-extern int ml_cpu_block_use(ml_cpu_block *p);
-
-/* defined in cpu_ctrl.S */
-extern void ml_halt_cpu(void);
-extern ml_cpu_block *ml_this_cpu(void);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

arch/x86_64/include/mango/machine/init.h

@@ -1,26 +0,0 @@
-#ifndef MANGO_X86_64_INIT_H_
-#define MANGO_X86_64_INIT_H_
-
-#include <stdint.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define __X2(x) #x
-#define __X(x) __X2(x)
-
-#define __define_initcall(fn, id) \
-	static initcall_t __initcall_##fn##id __used \
-	__section(".initcall" __X(id) ".init") = (fn)
-
-extern int ml_init(uintptr_t arg);
-
-extern const struct framebuffer_varinfo *bootfb_varinfo(void);
-extern const struct framebuffer_fixedinfo *bootfb_fixedinfo(void);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

arch/x86_64/include/mango/machine/irq.h

@@ -1,5 +0,0 @@
-#ifndef MANGO_X86_64_IRQ_H_
-#define MANGO_X86_64_IRQ_H_
-
-
-#endif

arch/x86_64/include/mango/machine/panic.h

@@ -1,12 +0,0 @@
-#ifndef MANGO_X86_64_PANIC_H_
-#define MANGO_X86_64_PANIC_H_
-
-#include <stdint.h>
-
-struct cpu_context;
-
-extern void ml_print_cpu_state(struct cpu_context *ctx);
-extern void ml_print_stack_trace(uintptr_t ip);
-extern void ml_print_stack_trace_irq(struct cpu_context *ctx);
-
-#endif

arch/x86_64/include/mango/machine/pmap.h

@@ -1,9 +0,0 @@
-#ifndef MANGO_X86_64_PMAP_H_
-#define MANGO_X86_64_PMAP_H_
-
-#include <arch/paging.h>
-
-typedef pml4t_ptr_t ml_pmap_t;
-typedef uint64_t ml_pfn_t;
-
-#endif

arch/x86_64/include/mango/machine/thread.h

@@ -1,10 +0,0 @@
-#ifndef MANGO_X86_64_THREAD_H_
-#define MANGO_X86_64_THREAD_H_
-
-#include <mango/sched.h>
-
-extern void switch_to(struct thread *from, struct thread *to);
-extern void prepare_stack(uintptr_t ip, uintptr_t *sp);
-extern void user_jump(uintptr_t ip, uintptr_t sp);
-
-#endif

arch/x86_64/init.c

@@ -2,22 +2,36 @@
 #include <arch/pit.h>
 #include <arch/serial.h>
 #include <arch/vgacon.h>
-#include <mango/arg.h>
+#include <kernel/arg.h>
-#include <mango/clock.h>
+#include <kernel/bsp.h>
-#include <mango/console.h>
+#include <kernel/clock.h>
-#include <mango/cpu.h>
+#include <kernel/console.h>
-#include <mango/init.h>
+#include <kernel/cpu.h>
-#include <mango/libc/stdio.h>
+#include <kernel/init.h>
-#include <mango/machine/cpu.h>
+#include <kernel/libc/stdio.h>
-#include <mango/memblock.h>
+#include <kernel/machine/cpu.h>
-#include <mango/object.h>
+#include <kernel/memblock.h>
-#include <mango/percpu.h>
+#include <kernel/object.h>
-#include <mango/pmap.h>
+#include <kernel/percpu.h>
-#include <mango/printk.h>
+#include <kernel/pmap.h>
-#include <mango/types.h>
+#include <kernel/printk.h>
-#include <mango/vm.h>
+#include <kernel/types.h>
+#include <kernel/vm.h>
 
-#define PTR32(x) ((void *)((uintptr_t)(x)))
+#define PTR32(x)            ((void *)((uintptr_t)(x)))
+
+/* the physical address of the start of the memblock heap.
+ * this is an arbirary value; the heap can start anywhere in memory.
+ * any reserved areas of memory (the kernel, bsp, bios data, etc) are
+ * automatically taken into account.
+ * HOWEVER, this value will dictate how much physical memory is required for
+ * the kernel to boot successfully.
+ * the value of 16MiB (0x1000000) means that all heap allocations will be
+ * above 16MiB, leaving the area below free for DMA operations.
+ * this value CAN be reduced all the way to zero to minimise the amount of
+ * memory required to boot, but this may leave you with no DMA memory available.
+ */
+#define MEMBLOCK_HEAP_START 0x1000000
 
 static ml_cpu_block g_bootstrap_cpu = {0};
 
@@ -30,9 +44,9 @@ static void bootstrap_cpu_init(void)
 	ml_cpu_block_use(&g_bootstrap_cpu);
 }
 
-static void early_vm_init(void)
+static void early_vm_init(uintptr_t reserve_end)
 {
-	uintptr_t alloc_start = VM_KERNEL_VOFFSET;
+	uintptr_t alloc_start = VM_KERNEL_VOFFSET + MEMBLOCK_HEAP_START;
 	/* boot code mapped 2 GiB of memory from
 	   VM_KERNEL_VOFFSET */
 	uintptr_t alloc_end = VM_KERNEL_VOFFSET + 0x7fffffff;
@@ -42,10 +56,10 @@ static void early_vm_init(void)
 	       alloc_start,
 	       alloc_end);
 
-	memblock_reserve(0x00, (uintptr_t)__pend);
+	memblock_reserve(0x00, reserve_end);
 	printk("memblock: reserved bios+kernel at [0x%016llx-0x%016llx]",
 	       0,
-	       (uintptr_t)__pend);
+	       reserve_end);
 }
 
 void early_console_init(void)
@@ -70,6 +84,23 @@ static void use_uniprocessor_topology(void)
 	cpu_set_online(0);
 }
 
+static void find_bsp(multiboot_info_t *mb, struct boot_module *out)
+{
+	memset(out, 0x0, sizeof *out);
+
+	printk("modules=%u: %llx", mb->mods_count, mb->mods_addr);
+
+	multiboot_module_t *mods = PTR32(mb->mods_addr);
+	size_t nr_mods = mb->mods_count;
+
+	if (nr_mods < 1) {
+		return;
+	}
+
+	out->mod_base = mods[0].mod_start;
+	out->mod_size = mods[0].mod_end - mods[0].mod_start;
+}
+
 int ml_init(uintptr_t arg)
 {
 	multiboot_info_t *mb = (multiboot_info_t *)arg;
@@ -83,7 +114,16 @@ int ml_init(uintptr_t arg)
 
 	print_kernel_banner();
 
-	early_vm_init();
+	struct boot_module bsp;
+	find_bsp(mb, &bsp);
+	bsp_set_location(&bsp);
+
+	uintptr_t reserve_end = (uintptr_t)__pend;
+	if (bsp.mod_base + bsp.mod_size > reserve_end) {
+		reserve_end = bsp.mod_base + bsp.mod_size;
+	}
+
+	early_vm_init(reserve_end);
 
 	e820_scan(PTR32(mb->mmap_addr), mb->mmap_length);
 
@@ -102,16 +142,20 @@ int ml_init(uintptr_t arg)
 	put_cpu(this_cpu);
 
 	struct vm_zone_descriptor vm_zones[] = {
-		{.zd_id = VM_ZONE_DMA,
+		{
-	         .zd_node = 0,
+			.zd_id = VM_ZONE_DMA,
-	         .zd_name = "dma",
+			.zd_node = 0,
-	         .zd_base = 0x00,
+			.zd_name = "dma",
-	         .zd_limit = 0xffffff},
+			.zd_base = 0x00,
-		{.zd_id = VM_ZONE_NORMAL,
+			.zd_limit = 0xffffff,
-	         .zd_node = 0,
+		},
-	         .zd_name = "normal",
+		{
-	         .zd_base = 0x1000000,
+			.zd_id = VM_ZONE_NORMAL,
-	         .zd_limit = UINTPTR_MAX},
+			.zd_node = 0,
+			.zd_name = "normal",
+			.zd_base = 0x1000000,
+			.zd_limit = UINTPTR_MAX,
+		},
 	};
 
 	vm_bootstrap(vm_zones, sizeof vm_zones / sizeof vm_zones[0]);

arch/x86_64/initcall.c

@@ -1,4 +1,4 @@
-#include <mango/init.h>
+#include <kernel/init.h>
 
 extern char __initcall0_start[];
 extern char __initcall1_start[];

arch/x86_64/irq.c

@@ -1,14 +1,22 @@
 #include <arch/irq.h>
+#include <arch/msr.h>
 #include <arch/ports.h>
-#include <mango/cpu.h>
+#include <kernel/cpu.h>
-#include <mango/libc/string.h>
+#include <kernel/libc/string.h>
-#include <mango/machine/cpu.h>
+#include <kernel/machine/cpu.h>
-#include <mango/machine/irq.h>
+#include <kernel/machine/irq.h>
-#include <mango/panic.h>
+#include <kernel/panic.h>
-#include <mango/sched.h>
+#include <kernel/sched.h>
+#include <kernel/syscall.h>
 #include <stddef.h>
 
-#define MAX_ISR_HANDLERS 16
+#define MAX_ISR_HANDLERS  16
+
+#define PF_PRESENT        0x01u
+#define PF_WRITE          0x02u
+#define PF_USER           0x04u
+#define PF_RESERVED_WRITE 0x08u
+#define PF_IFETCH         0x10u
 
 extern void syscall_gate(void);
 extern uintptr_t pf_faultptr(void);
@@ -20,6 +28,27 @@ static struct idt idt;
 static int idt_initialised = 0;
 static uintptr_t int_entry_points[NR_IDT_ENTRIES];
 
+static void set_syscall_gate(uintptr_t rip)
+{
+	uint64_t user_cs = 0x13;
+	uint64_t kernel_cs = 0x08;
+
+	uintptr_t star_reg = 0xC0000081;
+	uintptr_t lstar_reg = 0xC0000082;
+	uintptr_t sfmask_reg = 0xC0000084;
+
+	uint64_t selectors = 0;
+	selectors |= (user_cs) << 48;
+	selectors |= (kernel_cs) << 32;
+
+	/* disable interrupts */
+	uint64_t flag_mask = 0x200;
+
+	wrmsr(star_reg, selectors);
+	wrmsr(lstar_reg, rip);
+	wrmsr(sfmask_reg, flag_mask);
+}
+
 static void set_idt_gate(
 	struct idt *idt,
 	uint8_t index,
@@ -39,7 +68,7 @@ static void set_idt_gate(
 	idt->i_entries[index].reserved = 0;
 }
 
-static void gpf_handler(struct cpu_context *regs)
+static void gpf_handler(struct ml_cpu_context *regs)
 {
 	int ext = regs->err_no & 1;
 	int table = (regs->err_no >> 1) & 0x03;
@@ -55,43 +84,33 @@ static void gpf_handler(struct cpu_context *regs)
 		regs->rip);
 }
 
-static void pf_handler(struct cpu_context *regs)
+static void pf_handler(struct ml_cpu_context *regs)
 {
+	enum pmap_fault_flags fault_flags = 0;
+
+	(regs->err_no & PF_PRESENT) && (fault_flags |= PMAP_FAULT_PRESENT);
+	(regs->err_no & PF_USER) && (fault_flags |= PMAP_FAULT_USER);
+	(regs->err_no & PF_WRITE) && (fault_flags |= PMAP_FAULT_WRITE);
+	(regs->err_no & PF_IFETCH) && (fault_flags |= PMAP_FAULT_IFETCH);
+	(regs->err_no & PF_RESERVED_WRITE)
+		&& (fault_flags |= PMAP_FAULT_BADCFG);
+
+	virt_addr_t fault_ptr = pf_faultptr();
+
+	kern_status_t status = pmap_handle_fault(fault_ptr, fault_flags);
+
+	if (status == KERN_OK) {
+		return;
+	}
+
 	panic_irq(
 		regs,
 		"page fault (%016llx %016llx %016llx)",
-		pf_faultptr(),
+		fault_ptr,
 		regs->rip,
 		regs->err_no);
 }
 
-#if 0
-static void set_syscall_gate(uintptr_t rip)
-{
-	/* sysret adds 0x10 to this to get cs, and 0x8 to get ss
-	 * note that the CPU should force the RPL to 3 when loading
-	 * the selector by using user_cs | 3. However, this doesn't happen
-	 * in certain scenarios (specifically, QEMU + KVM on a Ryzen 5 1600X). */
-	uint64_t user_cs = 0x13;
-	uint64_t kernel_cs = 0x8;
-
-	uintptr_t star_reg = 0xC0000081;
-	uintptr_t lstar_reg = 0xC0000082;
-	uintptr_t sfmask_reg = 0xC0000084;
-
-	uint64_t selectors = 0;
-	selectors |= (user_cs) << 48;
-	selectors |= (kernel_cs) << 32;
-
-	/* disable interrupts */
-	uint64_t flag_mask = 0x200;
-
-	write_msr(star_reg, selectors);
-	write_msr(lstar_reg, rip);
-	write_msr(sfmask_reg, flag_mask);
-}
-#endif
-
 static void init_pic(void)
 {
 	// Remap the PIC
@@ -129,6 +148,8 @@ int idt_init(struct idt_ptr *ptr)
 		init_global_idt();
 	}
 
+	set_syscall_gate((uintptr_t)syscall_gate);
+
 	ptr->i_limit = sizeof(idt) - 1;
 	ptr->i_base = (uintptr_t)&idt;
 
@@ -141,7 +162,7 @@ int idt_load(struct idt_ptr *ptr)
 	return 0;
 }
 
-void isr_dispatch(struct cpu_context *regs)
+void isr_dispatch(struct ml_cpu_context *regs)
 {
 	int_hook h = isr_handlers[regs->int_no];
 	if (h) {
@@ -160,7 +181,7 @@ void irq_ack(unsigned int vec)
 	outportb(0x20, 0x20);
 }
 
-void irq_dispatch(struct cpu_context *regs)
+void irq_dispatch(struct ml_cpu_context *regs)
 {
 	end_charge_period();
 
@@ -178,8 +199,40 @@ void irq_dispatch(struct cpu_context *regs)
 	start_charge_period();
 }
 
-void syscall_dispatch(struct cpu_context *regs)
+void syscall_dispatch(struct ml_cpu_context *regs)
 {
+	unsigned int sysid = regs->rax;
+	virt_addr_t syscall_impl = syscall_get_function(sysid);
+
+	if (syscall_impl == 0) {
+		regs->rax = KERN_UNSUPPORTED;
+		return;
+	}
+
+#define SYSCALL_SIGNATURE(...)                                                 \
+	intptr_t (*__VA_ARGS__)(                                               \
+		uintptr_t,                                                     \
+		uintptr_t,                                                     \
+		uintptr_t,                                                     \
+		uintptr_t,                                                     \
+		uintptr_t,                                                     \
+		uintptr_t,                                                     \
+		uintptr_t,                                                     \
+		uintptr_t)
+
+	SYSCALL_SIGNATURE(fn) = (SYSCALL_SIGNATURE())syscall_impl;
+
+	ml_int_enable();
+	regs->rax
+		= fn(regs->rdi,
+	             regs->rsi,
+	             regs->rdx,
+	             regs->r12,
+	             regs->r8,
+	             regs->r9,
+	             regs->r13,
+	             regs->r14);
+	ml_int_disable();
 }
 
 void hook_irq(enum irq_vector vec, struct irq_hook *hook)
@@ -194,263 +247,263 @@ void unhook_irq(enum irq_vector vec, struct irq_hook *hook)
 	queue_delete(hook_queue, &hook->irq_entry);
 }
 
-extern void _isr0();
+extern void _isr0(void);
-extern void _isr1();
+extern void _isr1(void);
-extern void _isr2();
+extern void _isr2(void);
-extern void _isr3();
+extern void _isr3(void);
-extern void _isr4();
+extern void _isr4(void);
-extern void _isr5();
+extern void _isr5(void);
-extern void _isr6();
+extern void _isr6(void);
-extern void _isr7();
+extern void _isr7(void);
-extern void _isr8();
+extern void _isr8(void);
-extern void _isr9();
+extern void _isr9(void);
-extern void _isr10();
+extern void _isr10(void);
-extern void _isr11();
+extern void _isr11(void);
-extern void _isr12();
+extern void _isr12(void);
-extern void _isr13();
+extern void _isr13(void);
-extern void _isr14();
+extern void _isr14(void);
-extern void _isr15();
+extern void _isr15(void);
-extern void _isr16();
+extern void _isr16(void);
-extern void _isr17();
+extern void _isr17(void);
-extern void _isr18();
+extern void _isr18(void);
-extern void _isr19();
+extern void _isr19(void);
-extern void _isr20();
+extern void _isr20(void);
-extern void _isr21();
+extern void _isr21(void);
-extern void _isr22();
+extern void _isr22(void);
-extern void _isr23();
+extern void _isr23(void);
-extern void _isr24();
+extern void _isr24(void);
-extern void _isr25();
+extern void _isr25(void);
-extern void _isr26();
+extern void _isr26(void);
-extern void _isr27();
+extern void _isr27(void);
-extern void _isr28();
+extern void _isr28(void);
-extern void _isr29();
+extern void _isr29(void);
-extern void _isr30();
+extern void _isr30(void);
-extern void _isr31();
+extern void _isr31(void);
 
-extern void _irq0();
+extern void _irq0(void);
-extern void _irq1();
+extern void _irq1(void);
-extern void _irq2();
+extern void _irq2(void);
-extern void _irq3();
+extern void _irq3(void);
-extern void _irq4();
+extern void _irq4(void);
-extern void _irq5();
+extern void _irq5(void);
-extern void _irq6();
+extern void _irq6(void);
-extern void _irq7();
+extern void _irq7(void);
-extern void _irq8();
+extern void _irq8(void);
-extern void _irq9();
+extern void _irq9(void);
-extern void _irq10();
+extern void _irq10(void);
-extern void _irq11();
+extern void _irq11(void);
-extern void _irq12();
+extern void _irq12(void);
-extern void _irq13();
+extern void _irq13(void);
-extern void _irq14();
+extern void _irq14(void);
-extern void _irq15();
+extern void _irq15(void);
-extern void _irq16();
+extern void _irq16(void);
-extern void _irq17();
+extern void _irq17(void);
-extern void _irq18();
+extern void _irq18(void);
-extern void _irq19();
+extern void _irq19(void);
-extern void _irq20();
+extern void _irq20(void);
-extern void _irq21();
+extern void _irq21(void);
-extern void _irq22();
+extern void _irq22(void);
-extern void _irq23();
+extern void _irq23(void);
-extern void _irq24();
+extern void _irq24(void);
-extern void _irq25();
+extern void _irq25(void);
-extern void _irq26();
+extern void _irq26(void);
-extern void _irq27();
+extern void _irq27(void);
-extern void _irq28();
+extern void _irq28(void);
-extern void _irq29();
+extern void _irq29(void);
-extern void _irq30();
+extern void _irq30(void);
-extern void _irq31();
+extern void _irq31(void);
-extern void _irq32();
+extern void _irq32(void);
-extern void _irq33();
+extern void _irq33(void);
-extern void _irq34();
+extern void _irq34(void);
-extern void _irq35();
+extern void _irq35(void);
-extern void _irq36();
+extern void _irq36(void);
-extern void _irq37();
+extern void _irq37(void);
-extern void _irq38();
+extern void _irq38(void);
-extern void _irq39();
+extern void _irq39(void);
-extern void _irq40();
+extern void _irq40(void);
-extern void _irq41();
+extern void _irq41(void);
-extern void _irq42();
+extern void _irq42(void);
-extern void _irq43();
+extern void _irq43(void);
-extern void _irq44();
+extern void _irq44(void);
-extern void _irq45();
+extern void _irq45(void);
-extern void _irq46();
+extern void _irq46(void);
-extern void _irq47();
+extern void _irq47(void);
-extern void _irq48();
+extern void _irq48(void);
-extern void _irq49();
+extern void _irq49(void);
-extern void _irq50();
+extern void _irq50(void);
-extern void _irq51();
+extern void _irq51(void);
-extern void _irq52();
+extern void _irq52(void);
-extern void _irq53();
+extern void _irq53(void);
-extern void _irq54();
+extern void _irq54(void);
-extern void _irq55();
+extern void _irq55(void);
-extern void _irq56();
+extern void _irq56(void);
-extern void _irq57();
+extern void _irq57(void);
-extern void _irq58();
+extern void _irq58(void);
-extern void _irq59();
+extern void _irq59(void);
-extern void _irq60();
+extern void _irq60(void);
-extern void _irq61();
+extern void _irq61(void);
-extern void _irq62();
+extern void _irq62(void);
-extern void _irq63();
+extern void _irq63(void);
-extern void _irq64();
+extern void _irq64(void);
-extern void _irq65();
+extern void _irq65(void);
-extern void _irq66();
+extern void _irq66(void);
-extern void _irq67();
+extern void _irq67(void);
-extern void _irq68();
+extern void _irq68(void);
-extern void _irq69();
+extern void _irq69(void);
-extern void _irq70();
+extern void _irq70(void);
-extern void _irq71();
+extern void _irq71(void);
-extern void _irq72();
+extern void _irq72(void);
-extern void _irq73();
+extern void _irq73(void);
-extern void _irq74();
+extern void _irq74(void);
-extern void _irq75();
+extern void _irq75(void);
-extern void _irq76();
+extern void _irq76(void);
-extern void _irq77();
+extern void _irq77(void);
-extern void _irq78();
+extern void _irq78(void);
-extern void _irq79();
+extern void _irq79(void);
-extern void _irq80();
+extern void _irq80(void);
-extern void _irq81();
+extern void _irq81(void);
-extern void _irq82();
+extern void _irq82(void);
-extern void _irq83();
+extern void _irq83(void);
-extern void _irq84();
+extern void _irq84(void);
-extern void _irq85();
+extern void _irq85(void);
-extern void _irq86();
+extern void _irq86(void);
-extern void _irq87();
+extern void _irq87(void);
-extern void _irq88();
+extern void _irq88(void);
-extern void _irq89();
+extern void _irq89(void);
-extern void _irq90();
+extern void _irq90(void);
-extern void _irq91();
+extern void _irq91(void);
-extern void _irq92();
+extern void _irq92(void);
-extern void _irq93();
+extern void _irq93(void);
-extern void _irq94();
+extern void _irq94(void);
-extern void _irq95();
+extern void _irq95(void);
-extern void _irq96();
+extern void _irq96(void);
-extern void _irq97();
+extern void _irq97(void);
-extern void _irq98();
+extern void _irq98(void);
-extern void _irq99();
+extern void _irq99(void);
-extern void _irq100();
+extern void _irq100(void);
-extern void _irq101();
+extern void _irq101(void);
-extern void _irq102();
+extern void _irq102(void);
-extern void _irq103();
+extern void _irq103(void);
-extern void _irq104();
+extern void _irq104(void);
-extern void _irq105();
+extern void _irq105(void);
-extern void _irq106();
+extern void _irq106(void);
-extern void _irq107();
+extern void _irq107(void);
-extern void _irq108();
+extern void _irq108(void);
-extern void _irq109();
+extern void _irq109(void);
-extern void _irq110();
+extern void _irq110(void);
-extern void _irq111();
+extern void _irq111(void);
-extern void _irq112();
+extern void _irq112(void);
-extern void _irq113();
+extern void _irq113(void);
-extern void _irq114();
+extern void _irq114(void);
-extern void _irq115();
+extern void _irq115(void);
-extern void _irq116();
+extern void _irq116(void);
-extern void _irq117();
+extern void _irq117(void);
-extern void _irq118();
+extern void _irq118(void);
-extern void _irq119();
+extern void _irq119(void);
-extern void _irq120();
+extern void _irq120(void);
-extern void _irq121();
+extern void _irq121(void);
-extern void _irq122();
+extern void _irq122(void);
-extern void _irq123();
+extern void _irq123(void);
-extern void _irq124();
+extern void _irq124(void);
-extern void _irq125();
+extern void _irq125(void);
-extern void _irq126();
+extern void _irq126(void);
-extern void _irq127();
+extern void _irq127(void);
-extern void _irq128();
+extern void _irq128(void);
-extern void _irq129();
+extern void _irq129(void);
-extern void _irq130();
+extern void _irq130(void);
-extern void _irq131();
+extern void _irq131(void);
-extern void _irq132();
+extern void _irq132(void);
-extern void _irq133();
+extern void _irq133(void);
-extern void _irq134();
+extern void _irq134(void);
-extern void _irq135();
+extern void _irq135(void);
-extern void _irq136();
+extern void _irq136(void);
-extern void _irq137();
+extern void _irq137(void);
-extern void _irq138();
+extern void _irq138(void);
-extern void _irq139();
+extern void _irq139(void);
-extern void _irq140();
+extern void _irq140(void);
-extern void _irq141();
+extern void _irq141(void);
-extern void _irq142();
+extern void _irq142(void);
-extern void _irq143();
+extern void _irq143(void);
-extern void _irq144();
+extern void _irq144(void);
-extern void _irq145();
+extern void _irq145(void);
-extern void _irq146();
+extern void _irq146(void);
-extern void _irq147();
+extern void _irq147(void);
-extern void _irq148();
+extern void _irq148(void);
-extern void _irq149();
+extern void _irq149(void);
-extern void _irq150();
+extern void _irq150(void);
-extern void _irq151();
+extern void _irq151(void);
-extern void _irq152();
+extern void _irq152(void);
-extern void _irq153();
+extern void _irq153(void);
-extern void _irq154();
+extern void _irq154(void);
-extern void _irq155();
+extern void _irq155(void);
-extern void _irq156();
+extern void _irq156(void);
-extern void _irq157();
+extern void _irq157(void);
-extern void _irq158();
+extern void _irq158(void);
-extern void _irq159();
+extern void _irq159(void);
-extern void _irq160();
+extern void _irq160(void);
-extern void _irq161();
+extern void _irq161(void);
-extern void _irq162();
+extern void _irq162(void);
-extern void _irq163();
+extern void _irq163(void);
-extern void _irq164();
+extern void _irq164(void);
-extern void _irq165();
+extern void _irq165(void);
-extern void _irq166();
+extern void _irq166(void);
-extern void _irq167();
+extern void _irq167(void);
-extern void _irq168();
+extern void _irq168(void);
-extern void _irq169();
+extern void _irq169(void);
-extern void _irq170();
+extern void _irq170(void);
-extern void _irq171();
+extern void _irq171(void);
-extern void _irq172();
+extern void _irq172(void);
-extern void _irq173();
+extern void _irq173(void);
-extern void _irq174();
+extern void _irq174(void);
-extern void _irq175();
+extern void _irq175(void);
-extern void _irq176();
+extern void _irq176(void);
-extern void _irq177();
+extern void _irq177(void);
-extern void _irq178();
+extern void _irq178(void);
-extern void _irq179();
+extern void _irq179(void);
-extern void _irq180();
+extern void _irq180(void);
-extern void _irq181();
+extern void _irq181(void);
-extern void _irq182();
+extern void _irq182(void);
-extern void _irq183();
+extern void _irq183(void);
-extern void _irq184();
+extern void _irq184(void);
-extern void _irq185();
+extern void _irq185(void);
-extern void _irq186();
+extern void _irq186(void);
-extern void _irq187();
+extern void _irq187(void);
-extern void _irq188();
+extern void _irq188(void);
-extern void _irq189();
+extern void _irq189(void);
-extern void _irq190();
+extern void _irq190(void);
-extern void _irq191();
+extern void _irq191(void);
-extern void _irq192();
+extern void _irq192(void);
-extern void _irq193();
+extern void _irq193(void);
-extern void _irq194();
+extern void _irq194(void);
-extern void _irq195();
+extern void _irq195(void);
-extern void _irq196();
+extern void _irq196(void);
-extern void _irq197();
+extern void _irq197(void);
-extern void _irq198();
+extern void _irq198(void);
-extern void _irq199();
+extern void _irq199(void);
-extern void _irq200();
+extern void _irq200(void);
-extern void _irq201();
+extern void _irq201(void);
-extern void _irq202();
+extern void _irq202(void);
-extern void _irq203();
+extern void _irq203(void);
-extern void _irq204();
+extern void _irq204(void);
-extern void _irq205();
+extern void _irq205(void);
-extern void _irq206();
+extern void _irq206(void);
-extern void _irq207();
+extern void _irq207(void);
-extern void _irq208();
+extern void _irq208(void);
-extern void _irq209();
+extern void _irq209(void);
-extern void _irq210();
+extern void _irq210(void);
-extern void _irq211();
+extern void _irq211(void);
-extern void _irq212();
+extern void _irq212(void);
-extern void _irq213();
+extern void _irq213(void);
-extern void _irq214();
+extern void _irq214(void);
-extern void _irq215();
+extern void _irq215(void);
-extern void _irq216();
+extern void _irq216(void);
-extern void _irq217();
+extern void _irq217(void);
-extern void _irq218();
+extern void _irq218(void);
-extern void _irq219();
+extern void _irq219(void);
-extern void _irq220();
+extern void _irq220(void);
-extern void _irq221();
+extern void _irq221(void);
-extern void _irq222();
+extern void _irq222(void);
-extern void _irq223();
+extern void _irq223(void);
 
 static uintptr_t int_entry_points[NR_IDT_ENTRIES] = {
 	[0] = (uintptr_t)_isr0,     [1] = (uintptr_t)_isr1,

arch/x86_64/irqvec.S

@@ -332,80 +332,115 @@ IRQ 223, 255
 
 isr_common_stub:
 	PUSH_REGS
-	
+
+	# When ISR occurs in Ring 3, CPU sets %ss (and other non-code selectors)
+	# to 0.
+	mov %ss, %ax
+	cmp $0, %ax
+	jne isr_skipgs1
+
+	mov $0x10, %ax
+	mov %ax, %ss
+	swapgs
+
+isr_skipgs1:
 	mov %rsp, %rdi
 	call isr_dispatch
-	
+
 	POP_REGS
 	add $16, %rsp
+
+	cmpq $0x1b, 32(%rsp)
+	jne isr_skipgs2
+
+	swapgs
+
+isr_skipgs2:
+
 	iretq
-	
+
 
 	.global irq_common_stub
 	.type irq_common_stub, @function
 
 irq_common_stub:
 	PUSH_REGS
-	
+
+	# When IRQ occurs in Ring 3, CPU sets %ss (and other non-code selectors)
+	# to 0.
+	mov %ss, %ax
+	cmp $0, %ax
+	jne irq_skipgs1
+
+	mov $0x10, %ax
+	mov %ax, %ss
+	swapgs
+
+irq_skipgs1:
 	mov %rsp, %rdi
 	call irq_dispatch
-	
+
+
 	POP_REGS
 	add $16, %rsp
+
+	cmpq $0x1b, 32(%rsp)
+	jne isr_skipgs2
+
+	swapgs
+
+irq_skipgs2:
+
 	iretq
-	
+
 
 	.global syscall_gate
 	.type syscall_gate, @function
-	
+
 	.extern syscall_dispatch
 	.type syscall_dispatch, @function
-	
+
 syscall_gate:
 	swapgs
-	movq %rsp, %gs:20       # GS+20 = rsp2 in the current TSS block (user stack storage)
+	movq %rsp, %gs:94       # GS+20 = rsp2 in the current TSS block (user stack storage)
-	movq %gs:4, %rsp	# GS+4  = rsp0 in the current TSS block (per-thread kstack)
+	movq %gs:78, %rsp	# GS+4  = rsp0 in the current TSS block (per-thread kstack)
-	
+
-	# start building a pf_cpu_context
+	# start building a ml_cpu_context
 	pushq $0x1b
-	pushq %gs:20
+	pushq %gs:94
 	push %r11
 	push $0x23
 	push %rcx
-	
+
 	pushq $0
 	pushq $0x80
-	
+
 	PUSH_REGS
-	
+
 	mov %rsp, %rdi
-	
+
-	# switch back to user gs while in syscall_dispatch. Interrupts are enabled in syscall_dispatch,
-	# and if the task gets pre-empted, the incoming task will expect %gs to have its usermode value.
-	swapgs
-	
 	call syscall_dispatch
-	
+
 	POP_REGS
-	
+
 	add $16, %rsp
 	pop %rcx
 	add $8, %rsp
 	pop %r11
 	add $16, %rsp
-	
+
+	movq %gs:94, %rsp       # GS+20 = rsp2 in the current TSS block
+
 	swapgs
-	movq %gs:20, %rsp       # GS+20 = rsp2 in the current TSS block
+
-	swapgs
-	
 	# back to usermode
 	sysretq
-	
+
 
 	.global pf_faultptr
 	.type pf_faultptr, @function
 pf_faultptr:
 	mov %cr2, %rax
 	ret
-	
+
-	
+

arch/x86_64/panic.c

@@ -1,24 +1,25 @@
-#include "mango/machine/panic.h"
-#include "mango/vm.h"
-#include <mango/printk.h>
-#include <mango/libc/stdio.h>
 #include <arch/irq.h>
+#include <kernel/libc/stdio.h>
+#include <kernel/machine/cpu.h>
+#include <kernel/machine/panic.h>
+#include <kernel/printk.h>
+#include <kernel/vm.h>
 
-#define R_CF 0
+#define R_CF  0
-#define R_PF 2
+#define R_PF  2
-#define R_AF 4
+#define R_AF  4
-#define R_ZF 6
+#define R_ZF  6
-#define R_SF 7
+#define R_SF  7
-#define R_TF 8
+#define R_TF  8
-#define R_IF 9
+#define R_IF  9
-#define R_DF 10
+#define R_DF  10
-#define R_OF 11
+#define R_OF  11
-#define R_NT 14
+#define R_NT  14
-#define R_VM 17
+#define R_VM  17
-#define R_AC 18
+#define R_AC  18
 #define R_VIF 19
 #define R_VIP 20
-#define R_ID 21
+#define R_ID  21
 #define R_MAX 21
 
 struct stack_frame {
@@ -82,7 +83,11 @@ static void print_rflags(uintptr_t rflags)
 		if (rflags & (1 << i)) {
 			const char *name = pf_rfl_name(i);
 			if (name) {
-				buf_i += snprintf(buf + buf_i, sizeof(buf) - buf_i, " %s", name);
+				buf_i += snprintf(
+					buf + buf_i,
+					sizeof(buf) - buf_i,
+					" %s",
+					name);
 			}
 		}
 	}
@@ -91,30 +96,43 @@ static void print_rflags(uintptr_t rflags)
 	printk(buf);
 }
 
-void ml_print_cpu_state(struct cpu_context *ctx)
+void ml_print_cpu_state(struct ml_cpu_context *ctx)
 {
 	printk("cpu state:");
 	if (ctx) {
 		printk("  rax %016llx rbx %016llx rcx %016llx",
-		       ctx->rax, ctx->rbx, ctx->rcx);
+		       ctx->rax,
+		       ctx->rbx,
+		       ctx->rcx);
 		printk("  rdx %016llx rsi %016llx rdi %016llx",
-		       ctx->rdx, ctx->rsi, ctx->rdi);
+		       ctx->rdx,
+		       ctx->rsi,
+		       ctx->rdi);
 		printk("  rsp %016llx rbp %016llx r8  %016llx",
-		       ctx->rsp, ctx->rbp, ctx->r8);
+		       ctx->rsp,
+		       ctx->rbp,
+		       ctx->r8);
 		printk("  r9  %016llx r10 %016llx r11 %016llx",
-		       ctx->r9, ctx->r10, ctx->r11);
+		       ctx->r9,
+		       ctx->r10,
+		       ctx->r11);
 		printk("  r12 %016llx r13 %016llx r14 %016llx",
-		       ctx->r12, ctx->r13, ctx->r14);
+		       ctx->r12,
+		       ctx->r13,
+		       ctx->r14);
 		printk("  r15 %016llx rip %016llx cs  %04x ss %04x",
-		       ctx->r15, ctx->rip, ctx->cs, ctx->ss);
+		       ctx->r15,
+		       ctx->rip,
+		       ctx->cs,
+		       ctx->ss);
 		print_rflags(ctx->rflags);
 	}
 
 	uintptr_t cr0 = 0, cr2 = 0, cr3 = 0, cr4 = 0;
-	asm volatile("mov %%cr0, %%rax" : "=a" (cr0));
+	asm volatile("mov %%cr0, %%rax" : "=a"(cr0));
-	asm volatile("mov %%cr2, %%rax" : "=a" (cr2));
+	asm volatile("mov %%cr2, %%rax" : "=a"(cr2));
-	asm volatile("mov %%cr3, %%rax" : "=a" (cr3));
+	asm volatile("mov %%cr3, %%rax" : "=a"(cr3));
-	asm volatile("mov %%cr4, %%rax" : "=a" (cr4));
+	asm volatile("mov %%cr4, %%rax" : "=a"(cr4));
 	printk("  cr0 %016llx cr2 %016llx", cr0, cr2);
 	printk("  cr3 %016llx cr4 %016llx", cr3, cr4);
 }
@@ -124,22 +142,27 @@ static void print_stack_item(uintptr_t addr)
 	if (!addr) {
 		return;
 	}
-	
+
 	char buf[64];
 	size_t i = 0;
-	
+
 	i += snprintf(buf, sizeof(buf), "  [<%p>] ", addr);
-	
+
 	size_t offset = 0;
 	char name[128];
 	int found = -1;
-	
+
 	if (found == 0 && name[0] != '\0') {
-		i += snprintf(buf + i, sizeof(buf) - i, "%s+0x%lx", name, offset);
+		i += snprintf(
+			buf + i,
+			sizeof(buf) - i,
+			"%s+0x%lx",
+			name,
+			offset);
 	} else {
 		i += snprintf(buf + i, sizeof(buf) - i, "?");
 	}
-	
+
 	printk("%s", buf);
 }
 
@@ -147,20 +170,19 @@ static void print_stack_trace(uintptr_t ip, uintptr_t *bp)
 {
 	struct stack_frame *stk = (struct stack_frame *)bp;
 	printk("call trace:");
-	
+
 	print_stack_item(ip);
 
 	int max_frames = 10, current_frame = 0;
 	while (1) {
-		if (!vm_virt_to_phys(stk) ||
+		if (!vm_virt_to_phys(stk) || bp == NULL
-		    bp == NULL ||
+		    || current_frame > max_frames) {
-		    current_frame > max_frames) {
 			break;
 		}
-		
+
 		uintptr_t addr = stk->rip;
 		print_stack_item(addr);
-		
+
 		stk = (struct stack_frame *)stk->rbp;
 		current_frame++;
 	}
@@ -169,11 +191,11 @@ static void print_stack_trace(uintptr_t ip, uintptr_t *bp)
 void ml_print_stack_trace(uintptr_t ip)
 {
 	uintptr_t *bp;
-	asm volatile("mov %%rbp, %0" : "=r" (bp));
+	asm volatile("mov %%rbp, %0" : "=r"(bp));
 	print_stack_trace(ip, bp);
 }
 
-void ml_print_stack_trace_irq(struct cpu_context *ctx)
+void ml_print_stack_trace_irq(struct ml_cpu_context *ctx)
 {
 	print_stack_trace(ctx->rip, (uintptr_t *)ctx->rbp);
 }

arch/x86_64/pit.c

@@ -1,8 +1,8 @@
 #include <arch/irq.h>
 #include <arch/ports.h>
-#include <mango/clock.h>
+#include <kernel/clock.h>
-#include <mango/cpu.h>
+#include <kernel/cpu.h>
-#include <mango/printk.h>
+#include <kernel/printk.h>
 
 #define PIT_COUNTER0  0x40
 #define PIT_CMD       0x43

arch/x86_64/pmap.c

@@ -1,17 +1,21 @@
-#include <mango/compiler.h>
+#include <kernel/compiler.h>
-#include <mango/memblock.h>
+#include <kernel/libc/stdio.h>
-#include <mango/pmap.h>
+#include <kernel/memblock.h>
-#include <mango/printk.h>
+#include <kernel/pmap.h>
+#include <kernel/printk.h>
+#include <kernel/sched.h>
+#include <kernel/types.h>
+#include <kernel/vm-object.h>
+#include <kernel/vm-region.h>
+#include <kernel/vm.h>
 #include <mango/status.h>
-#include <mango/types.h>
-#include <mango/vm.h>
 
 /* some helpful datasize constants */
 #define C_1GiB          0x40000000ULL
 #define C_2GiB          (2 * C_1GiB)
 
 #define BAD_INDEX       ((unsigned int)-1)
-#define PTR_TO_ENTRY(x) (((x) & ~VM_PAGE_MASK) | PTE_PRESENT | PTE_RW)
+#define PTR_TO_ENTRY(x) (((x) & ~VM_PAGE_MASK) | PTE_PRESENT | PTE_RW | PTE_USR)
 #define ENTRY_TO_PTR(x) ((x) & ~VM_PAGE_MASK)
 
 #define PFN(x)          ((x) >> VM_PAGE_SHIFT)
@@ -39,7 +43,7 @@ static pmap_t alloc_pmap(void)
 	return vm_virt_to_phys(p);
 }
 
-static pte_t make_pte(pfn_t pfn, enum vm_prot prot, enum page_size size)
+static pte_t make_pte(pfn_t pfn, vm_prot_t prot, enum page_size size)
 {
 	pte_t v = pfn;
 
@@ -133,12 +137,11 @@ static void delete_pdir(phys_addr_t pd)
 
 static kern_status_t do_pmap_add(
 	pmap_t pmap,
-	void *p,
+	virt_addr_t pv,
 	pfn_t pfn,
-	enum vm_prot prot,
+	vm_prot_t prot,
 	enum page_size size)
 {
-	uintptr_t pv = (uintptr_t)p;
 	unsigned int pml4t_index = BAD_INDEX, pdpt_index = BAD_INDEX,
 		     pd_index = BAD_INDEX, pt_index = BAD_INDEX;
 
@@ -261,11 +264,11 @@ void pmap_bootstrap(void)
 	/* map 2GiB at the end of the address space to
 	   replace the mapping created by start_32 and allow access to
 	   the kernel and memblock-allocated data. */
-	uintptr_t vbase = VM_KERNEL_VOFFSET;
+	virt_addr_t vbase = VM_KERNEL_VOFFSET;
 	for (size_t i = 0; i < C_2GiB; i += hugepage_sz) {
 		do_pmap_add(
 			kernel_pmap,
-			(void *)(vbase + i),
+			vbase + i,
 			PFN(i),
 			VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXEC
 				| VM_PROT_SVR,
@@ -285,7 +288,7 @@ void pmap_bootstrap(void)
 	for (size_t i = 0; i < pmem_limit; i += hugepage_sz) {
 		do_pmap_add(
 			kernel_pmap,
-			(void *)(vbase + i),
+			vbase + i,
 			PFN(i),
 			VM_PROT_READ | VM_PROT_WRITE | VM_PROT_SVR
 				| VM_PROT_NOCACHE,
@@ -297,18 +300,83 @@ void pmap_bootstrap(void)
 
 pmap_t pmap_create(void)
 {
-	return 0;
+	pmap_t pmap = alloc_pmap();
+	pmap_t kernel_pmap = get_kernel_pmap();
+
+	struct pml4t *pml4t = vm_phys_to_virt(pmap);
+	struct pml4t *kernel_pml4t = vm_phys_to_virt(kernel_pmap);
+
+	for (unsigned int i = 256; i < 512; i++) {
+		pml4t->p_entries[i] = kernel_pml4t->p_entries[i];
+	}
+
+	return pmap;
 }
 
 void pmap_destroy(pmap_t pmap)
 {
 }
 
+static void log_fault(virt_addr_t fault_addr, enum pmap_fault_flags flags)
+{
+	char flag_str[128] = {0};
+	size_t p = 0;
+
+	if (flags & PMAP_FAULT_PRESENT) {
+		p += snprintf(flag_str + p, sizeof flag_str - p, " PRESENT");
+	} else {
+		p += snprintf(flag_str + p, sizeof flag_str - p, " MISSING");
+	}
+
+	if (flags & PMAP_FAULT_USER) {
+		p += snprintf(flag_str + p, sizeof flag_str - p, " USER");
+	} else {
+		p += snprintf(flag_str + p, sizeof flag_str - p, " SVR");
+	}
+
+	if (flags & PMAP_FAULT_WRITE) {
+		p += snprintf(flag_str + p, sizeof flag_str - p, " WRITE");
+	} else {
+		p += snprintf(flag_str + p, sizeof flag_str - p, " READ");
+	}
+
+	if (flags & PMAP_FAULT_IFETCH) {
+		p += snprintf(flag_str + p, sizeof flag_str - p, " IFETCH");
+	}
+
+	if (flags & PMAP_FAULT_BADCFG) {
+		p += snprintf(flag_str + p, sizeof flag_str - p, " BADCFG");
+	}
+
+	printk("pmap: fault at 0x%llx (%s)", fault_addr, flag_str);
+}
+
+kern_status_t pmap_handle_fault(
+	virt_addr_t fault_addr,
+	enum pmap_fault_flags flags)
+{
+	// log_fault(fault_addr, flags);
+
+	if (flags & PMAP_FAULT_PRESENT) {
+		return KERN_FATAL_ERROR;
+	}
+
+	struct task *task = current_task();
+	struct vm_region *space = task->t_address_space;
+
+	unsigned long lock_flags;
+	vm_region_lock_irqsave(space, &lock_flags);
+	kern_status_t status = vm_region_demand_map(space, fault_addr, flags);
+	vm_region_unlock_irqrestore(space, lock_flags);
+
+	return status;
+}
+
 kern_status_t pmap_add(
 	pmap_t pmap,
-	void *p,
+	virt_addr_t p,
 	pfn_t pfn,
-	enum vm_prot prot,
+	vm_prot_t prot,
 	enum pmap_flags flags)
 {
 	enum page_size ps = PS_4K;
@@ -321,21 +389,21 @@ kern_status_t pmap_add(
 
 kern_status_t pmap_add_block(
 	pmap_t pmap,
-	void *p,
+	virt_addr_t p,
 	pfn_t pfn,
 	size_t len,
-	enum vm_prot prot,
+	vm_prot_t prot,
 	enum pmap_flags flags)
 {
 	return KERN_OK;
 }
 
-kern_status_t pmap_remove(pmap_t pmap, void *p)
+kern_status_t pmap_remove(pmap_t pmap, virt_addr_t p)
 {
 	return KERN_OK;
 }
 
-kern_status_t pmap_remove_range(pmap_t pmap, void *p, size_t len)
+kern_status_t pmap_remove_range(pmap_t pmap, virt_addr_t p, size_t len)
 {
 	return KERN_OK;
 }

arch/x86_64/serial.c

@@ -1,8 +1,8 @@
 #include <arch/irq.h>
 #include <arch/ports.h>
 #include <arch/serial.h>
-#include <mango/libc/stdio.h>
+#include <kernel/libc/stdio.h>
-#include <mango/printk.h>
+#include <kernel/printk.h>
 
 #define COM1 0x3F8
 #define COM2 0x2F8

arch/x86_64/thread.c

@@ -1,12 +1,24 @@
-#include <mango/machine/thread.h>
+#include <kernel/machine/cpu.h>
+#include <kernel/machine/thread.h>
 
+#define MAX_REG_ARGS 6
+#define REG_ARG_0    rdi
+#define REG_ARG_1    rsi
+#define REG_ARG_2    rdx
+#define REG_ARG_3    rcx
+#define REG_ARG_4    r8
+#define REG_ARG_5    r9
+
+/* this is the context information restored by ml_thread_switch.
+ * since ml_thread_switch only jumps to kernel-mode, IRETQ isn't used,
+ * and the extra register values needed by IRETQ aren't present. */
 struct thread_ctx {
 	uint64_t r15, r14, r13, r12, r11, r10, r9, r8;
 	uint64_t rdi, rsi, rbp, unused_rsp, rbx, rdx, rcx, rax;
 	uint64_t rfl;
 } __packed;
 
-void prepare_stack(uintptr_t ip, uintptr_t *sp)
+void ml_thread_prepare_kernel_context(uintptr_t ip, uintptr_t *sp)
 {
 	(*sp) -= sizeof(uintptr_t);
 	uintptr_t *dest_ip = (uintptr_t *)(*sp);
@@ -18,3 +30,50 @@ void prepare_stack(uintptr_t ip, uintptr_t *sp)
 
 	ctx->rfl = 0x202;
 }
+
+extern kern_status_t ml_thread_prepare_user_context(
+	virt_addr_t ip,
+	virt_addr_t user_sp,
+	virt_addr_t *kernel_sp,
+	const uintptr_t *args,
+	size_t nr_args)
+{
+	(*kernel_sp) -= sizeof(struct ml_cpu_context);
+
+	struct ml_cpu_context *ctx = (struct ml_cpu_context *)(*kernel_sp);
+	memset(ctx, 0x0, sizeof *ctx);
+	ctx->rip = ip;
+	ctx->rsp = user_sp;
+	ctx->ss = 0x1b;
+	ctx->cs = 0x23;
+	ctx->rflags = 0x202;
+	ctx->rdi = 0;   // arg 0
+	ctx->rsi = 0;   // arg 1
+
+	for (size_t i = 0; i < nr_args; i++) {
+		switch (i) {
+		case 0:
+			ctx->REG_ARG_0 = args[i];
+			break;
+		case 1:
+			ctx->REG_ARG_1 = args[i];
+			break;
+		case 2:
+			ctx->REG_ARG_2 = args[i];
+			break;
+		case 3:
+			ctx->REG_ARG_3 = args[i];
+			break;
+		case 4:
+			ctx->REG_ARG_4 = args[i];
+			break;
+		case 5:
+			ctx->REG_ARG_5 = args[i];
+			break;
+		default:
+			return KERN_INVALID_ARGUMENT;
+		}
+	}
+
+	return KERN_OK;
+}

arch/x86_64/thread_switch.S

@@ -1,13 +1,13 @@
-	.code64
+.code64
 
-	.extern THREAD_sp
+.extern THREAD_sp
-	
+
-	.global switch_to
+.global ml_thread_switch
-	.type switch_to, @function
+.type ml_thread_switch, @function
 
 	// %rdi = (struct thread *) current thread.
 	// %rsi = (struct thread *) next thread.
-switch_to:
+ml_thread_switch:
 	pushfq
 
 	push %rax
@@ -26,7 +26,7 @@ switch_to:
 	push %r13
 	push %r14
 	push %r15
-	
+
 	movq %rsp, THREAD_sp(%rdi)
 	movq THREAD_sp(%rsi), %rsp
 
@@ -50,3 +50,28 @@ switch_to:
 	popfq
 
 	ret
+
+
+.global ml_thread_switch_user
+.type ml_thread_switch_user, @function
+ml_thread_switch_user:
+	pop %r15
+	pop %r14
+	pop %r13
+	pop %r12
+	pop %r11
+	pop %r10
+	pop %r9
+	pop %r8
+	pop %rdi
+	pop %rsi
+	pop %rbp
+	add $8, %rsp
+	pop %rbx
+	pop %rdx
+	pop %rcx
+	pop %rax
+
+	add $16, %rsp
+	swapgs
+	iretq

arch/x86_64/tss.c

@@ -0,0 +1,43 @@
+#include <arch/gdt.h>
+#include <arch/tss.h>
+#include <kernel/libc/string.h>
+
+static void tss_flush(int index)
+{
+	index *= sizeof(struct gdt_entry);
+	index |= 3;
+	asm volatile("mov %0, %%eax; ltr %%ax" ::"r"(index));
+}
+
+void tss_init(struct tss *tss, struct tss_ptr *ptr)
+{
+	memset(tss, 0x0, sizeof *tss);
+
+	ptr->tss_base = (uint64_t)tss;
+	ptr->tss_limit = (uint16_t)sizeof *tss;
+}
+
+void tss_load(struct tss *tss)
+{
+	tss_flush(TSS_GDT_INDEX);
+}
+
+virt_addr_t tss_get_kstack(struct tss *tss)
+{
+	return tss->rsp0;
+}
+
+virt_addr_t tss_get_ustack(struct tss *tss)
+{
+	return tss->rsp2;
+}
+
+void tss_set_kstack(struct tss *tss, virt_addr_t sp)
+{
+	tss->rsp0 = sp;
+}
+
+void tss_set_ustack(struct tss *tss, virt_addr_t sp)
+{
+	tss->rsp2 = sp;
+}

arch/x86_64/vga.c

@@ -1,9 +1,9 @@
 #include <arch/irq.h>
 #include <arch/ports.h>
 #include <arch/serial.h>
-#include <mango/libc/stdio.h>
+#include <kernel/libc/stdio.h>
-#include <mango/machine/vm.h>
+#include <kernel/machine/vm.h>
-#include <mango/printk.h>
+#include <kernel/printk.h>
 
 struct vga_console {
 	uint16_t *vga_framebuffer;

ds/bitmap.c

@@ -1,5 +1,5 @@
-#include <mango/libc/string.h>
+#include <kernel/bitmap.h>
-#include <mango/bitmap.h>
+#include <kernel/libc/string.h>
 
 void bitmap_zero(unsigned long *map, unsigned long nbits)
 {
@@ -25,7 +25,7 @@ void bitmap_set(unsigned long *map, unsigned long bit)
 void bitmap_clear(unsigned long *map, unsigned long bit)
 {
 	unsigned long index = bit / BITS_PER_WORD;
-	unsigned long offset = bit & (BITS_PER_WORD - 1);
+	unsigned long offset = (BITS_PER_WORD - bit - 1) & (BITS_PER_WORD - 1);
 	unsigned long mask = 1ul << offset;
 
 	map[index] &= ~mask;
@@ -38,7 +38,6 @@ bool bitmap_check(unsigned long *map, unsigned long bit)
 	unsigned long mask = 1ul << offset;
 
 	return (map[index] & mask) != 0 ? true : false;
-
 }
 
 unsigned int bitmap_count_set(unsigned long *map, unsigned long nbits)

ds/btree.c

@@ -57,7 +57,7 @@
    provide a comparator function.
 */ 
    
-#include <mango/btree.h>
+#include <kernel/btree.h>
 #include <stddef.h>
 
 #define MAX(a, b) ((a) > (b) ? (a) : (b))

ds/queue.c

@@ -1,4 +1,4 @@
-#include <mango/queue.h>
+#include <kernel/queue.h>
 
 size_t queue_length(struct queue *q)
 {

ds/ringbuffer.c

@@ -1,5 +1,5 @@
-#include <mango/ringbuffer.h>
+#include <kernel/ringbuffer.h>
-#include <mango/sched.h>
+#include <kernel/sched.h>
 
 size_t ringbuffer_unread(struct ringbuffer *ring_buffer)
 {

include/kernel/arg.h

@@ -1,8 +1,8 @@
-#ifndef MANGO_ARG_H_
+#ifndef KERNEL_ARG_H_
-#define MANGO_ARG_H_
+#define KERNEL_ARG_H_
 
+#include <mango/types.h>
 #include <stdbool.h>
-#include <mango/status.h>
 
 #define CMDLINE_MAX 4096
 

include/kernel/bitmap.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_BITMAP_H_
+#ifndef KERNEL_BITMAP_H_
-#define MANGO_BITMAP_H_
+#define KERNEL_BITMAP_H_
 
 #include <stdbool.h>
 

include/kernel/bsp.h

@@ -0,0 +1,38 @@
+#ifndef KERNEL_BSP_H_
+#define KERNEL_BSP_H_
+
+#include <kernel/compiler.h>
+#include <mango/status.h>
+#include <kernel/types.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#define BSP_MAGIC 0xcafebabe
+
+struct task;
+
+struct bsp_trailer {
+	/* these fields are stored in big endian in the package itself */
+	uint32_t bsp_magic;
+	uint64_t bsp_fs_offset;
+	uint32_t bsp_fs_len;
+	uint64_t bsp_exec_offset;
+	uint32_t bsp_exec_len;
+	uint64_t bsp_text_faddr, bsp_text_vaddr, bsp_text_size;
+	uint64_t bsp_data_faddr, bsp_data_vaddr, bsp_data_size;
+	uint64_t bsp_exec_entry;
+} __packed;
+
+struct bsp {
+	/* the values in this struct are stored in host byte order */
+	struct bsp_trailer bsp_trailer;
+	struct vm_object *bsp_vmo;
+};
+
+extern void bsp_set_location(const struct boot_module *mod);
+extern void bsp_get_location(struct boot_module *out);
+
+extern kern_status_t bsp_load(struct bsp *bsp, const struct boot_module *mod);
+extern kern_status_t bsp_launch_async(struct bsp *bsp, struct task *task);
+
+#endif

include/kernel/btree.h

@@ -20,99 +20,130 @@
     software without specific prior written permission.
  */
 
-#ifndef MANGO_BTREE_H_
+#ifndef KERNEL_BTREE_H_
-#define MANGO_BTREE_H_
+#define KERNEL_BTREE_H_
 
+#include <stdbool.h>
+#include <stddef.h>
 #include <stdint.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-/* if your custom structure contains a struct btree_node (i.e. it can be part of a btree),
+/* if your custom structure contains a struct btree_node (i.e. it can be part of
-   you can use this macro to convert a struct btree_node* to a your_type*
+   a btree), you can use this macro to convert a struct btree_node* to a
+   your_type*
 
-   @param t the name of your custom type (something that can be passed to offsetof)
+   @param t the name of your custom type (something that can be passed to
-   @param m the name of the struct btree_node member variable within your custom type.
+   offsetof)
-   @param v the struct btree_node pointer that you wish to convert. if this is NULL, NULL will be returned.
+   @param m the name of the struct btree_node member variable within your custom
+   type.
+   @param v the struct btree_node pointer that you wish to convert. if this is
+   NULL, NULL will be returned.
 */
-#define BTREE_CONTAINER(t, m, v) ((void *)((v) ? (uintptr_t)(v) - (offsetof(t, m)) : 0))
+#define BTREE_CONTAINER(t, m, v)                                               \
+	((void *)((v) ? (uintptr_t)(v) - (offsetof(t, m)) : 0))
 
 /* defines a simple node insertion function.
-   this function assumes that your nodes have simple integer keys that can be compared with the usual operators.
+   this function assumes that your nodes have simple integer keys that can be
+   compared with the usual operators.
 
    EXAMPLE:
    if you have a tree node type like this:
 
      struct my_tree_node {
              int key;
-	     struct btree_node base;
+             struct btree_node base;
      }
 
-   You would use the following call to generate an insert function for a tree with this node type:
+   You would use the following call to generate an insert function for a tree
+   with this node type:
 
-     BTREE_DEFINE_SIMPLE_INSERT(struct my_tree_node, base, key, my_tree_node_insert);
+     BTREE_DEFINE_SIMPLE_INSERT(struct my_tree_node, base, key,
+   my_tree_node_insert);
 
    Which would emit a function defined like:
 
-     static void my_tree_node_insert(struct btree *tree, struct my_tree_node *node);
+     static void my_tree_node_insert(struct btree *tree, struct my_tree_node
+   *node);
 
-   @param node_type your custom tree node type. usually a structure that contains a struct btree_node member.
+   @param node_type your custom tree node type. usually a structure that
-   @param container_node_member the name of the struct btree_node member variable within your custom type.
+   contains a struct btree_node member.
-   @param container_key_member the name of the key member variable within your custom type.
+   @param container_node_member the name of the struct btree_node member
+   variable within your custom type.
+   @param container_key_member the name of the key member variable within your
+   custom type.
    @param function_name the name of the function to generate.
 */
-#define BTREE_DEFINE_SIMPLE_INSERT(node_type, container_node_member, container_key_member, function_name) \
+#define BTREE_DEFINE_SIMPLE_INSERT(                                            \
-	void function_name(struct btree *tree, node_type *node)                                           \
+	node_type,                                                             \
-	{                                                                                                 \
+	container_node_member,                                                 \
-		if (!tree->b_root) {                                                                      \
+	container_key_member,                                                  \
-			tree->b_root = &node->container_node_member;                                      \
+	function_name)                                                         \
-			btree_insert_fixup(tree, &node->container_node_member);                           \
+	void function_name(struct btree *tree, node_type *node)                \
-			return;                                                                           \
+	{                                                                      \
-		}                                                                                         \
+		if (!tree->b_root) {                                           \
-                                                                                                          \
+			tree->b_root = &node->container_node_member;           \
-		struct btree_node *cur = tree->b_root;                                                    \
+			btree_insert_fixup(                                    \
-		while (1) {                                                                               \
+				tree,                                          \
-			node_type *cur_node = BTREE_CONTAINER(node_type, container_node_member, cur);     \
+				&node->container_node_member);                 \
-			struct btree_node *next = NULL;                                                   \
+			return;                                                \
-                                                                                                          \
+		}                                                              \
-			if (node->container_key_member > cur_node->container_key_member) {                \
+                                                                               \
-				next = btree_right(cur);                                                  \
+		struct btree_node *cur = tree->b_root;                         \
-                                                                                                          \
+		while (1) {                                                    \
-				if (!next) {                                                              \
+			node_type *cur_node = BTREE_CONTAINER(                 \
-					btree_put_right(cur, &node->container_node_member);               \
+				node_type,                                     \
-					break;                                                            \
+				container_node_member,                         \
-				}                                                                         \
+				cur);                                          \
-			} else if (node->container_key_member < cur_node->container_key_member) {         \
+			struct btree_node *next = NULL;                        \
-				next = btree_left(cur);                                                   \
+                                                                               \
-                                                                                                          \
+			if (node->container_key_member                         \
-				if (!next) {                                                              \
+			    > cur_node->container_key_member) {                \
-					btree_put_left(cur, &node->container_node_member);                \
+				next = btree_right(cur);                       \
-					break;                                                            \
+                                                                               \
-				}                                                                         \
+				if (!next) {                                   \
-			} else {                                                                          \
+					btree_put_right(                       \
-				return;                                                                   \
+						cur,                           \
-			}                                                                                 \
+						&node->container_node_member); \
-                                                                                                          \
+					break;                                 \
-			cur = next;                                                                       \
+				}                                              \
-		}                                                                                         \
+			} else if (                                            \
-                                                                                                          \
+				node->container_key_member                     \
-		btree_insert_fixup(tree, &node->container_node_member);                                   \
+				< cur_node->container_key_member) {            \
+				next = btree_left(cur);                        \
+                                                                               \
+				if (!next) {                                   \
+					btree_put_left(                        \
+						cur,                           \
+						&node->container_node_member); \
+					break;                                 \
+				}                                              \
+			} else {                                               \
+				return;                                        \
+			}                                                      \
+                                                                               \
+			cur = next;                                            \
+		}                                                              \
+                                                                               \
+		btree_insert_fixup(tree, &node->container_node_member);        \
 	}
 
 /* defines a node insertion function.
-   this function should be used for trees with complex node keys that cannot be directly compared.
+   this function should be used for trees with complex node keys that cannot be
-   a comparator for your keys must be supplied.
+   directly compared. a comparator for your keys must be supplied.
 
    EXAMPLE:
    if you have a tree node type like this:
 
      struct my_tree_node {
              complex_key_t key;
-	     struct btree_node base;
+             struct btree_node base;
      }
 
-   You would need to define a comparator function or macro with the following signature:
+   You would need to define a comparator function or macro with the following
+   signature:
 
      int my_comparator(struct my_tree_node *a, struct my_tree_node *b);
 
@@ -122,102 +153,136 @@ extern "C" {
      return  0   if a == b
      return  1   if a > b
 
-   You would use the following call to generate an insert function for a tree with this node type:
+   You would use the following call to generate an insert function for a tree
+   with this node type:
 
-     BTREE_DEFINE_INSERT(struct my_tree_node, base, key, my_tree_node_insert, my_comparator);
+     BTREE_DEFINE_INSERT(struct my_tree_node, base, key, my_tree_node_insert,
+   my_comparator);
 
    Which would emit a function defined like:
 
-     static void my_tree_node_insert(struct btree *tree, struct my_tree_node *node);
+     static void my_tree_node_insert(struct btree *tree, struct my_tree_node
+   *node);
 
-   @param node_type your custom tree node type. usually a structure that contains a struct btree_node member.
+   @param node_type your custom tree node type. usually a structure that
-   @param container_node_member the name of the struct btree_node member variable within your custom type.
+   contains a struct btree_node member.
-   @param container_key_member the name of the key member variable within your custom type.
+   @param container_node_member the name of the struct btree_node member
+   variable within your custom type.
+   @param container_key_member the name of the key member variable within your
+   custom type.
    @param function_name the name of the function to generate.
-   @param comparator the name of a comparator function or functional-macro that conforms to the
+   @param comparator the name of a comparator function or functional-macro that
-                     requirements listed above.
+   conforms to the requirements listed above.
 */
-#define BTREE_DEFINE_INSERT(node_type, container_node_member, container_key_member, function_name, comparator) \
+#define BTREE_DEFINE_INSERT(                                                   \
-	void function_name(struct btree *tree, node_type *node)                                                \
+	node_type,                                                             \
-	{                                                                                                      \
+	container_node_member,                                                 \
-		if (!tree->b_root) {                                                                           \
+	container_key_member,                                                  \
-			tree->b_root = &node->container_node_member;                                           \
+	function_name,                                                         \
-			btree_insert_fixup(tree, &node->container_node_member);                                \
+	comparator)                                                            \
-			return;                                                                                \
+	void function_name(struct btree *tree, node_type *node)                \
-		}                                                                                              \
+	{                                                                      \
-                                                                                                               \
+		if (!tree->b_root) {                                           \
-		struct btree_node *cur = tree->b_root;                                                         \
+			tree->b_root = &node->container_node_member;           \
-		while (1) {                                                                                    \
+			btree_insert_fixup(                                    \
-			node_type *cur_node = BTREE_CONTAINER(node_type, container_node_member, cur);          \
+				tree,                                          \
-			struct btree_node *next = NULL;                                                        \
+				&node->container_node_member);                 \
-			int cmp = comparator(node, cur_node);                                                  \
+			return;                                                \
-                                                                                                               \
+		}                                                              \
-			if (cmp == 1) {                                                                        \
+                                                                               \
-				next = btree_right(cur);                                                       \
+		struct btree_node *cur = tree->b_root;                         \
-                                                                                                               \
+		while (1) {                                                    \
-				if (!next) {                                                                   \
+			node_type *cur_node = BTREE_CONTAINER(                 \
-					btree_put_right(cur, &node->container_node_member);                    \
+				node_type,                                     \
-					break;                                                                 \
+				container_node_member,                         \
-				}                                                                              \
+				cur);                                          \
-			} else if (cmp == -1) {                                                                \
+			struct btree_node *next = NULL;                        \
-				next = btree_left(cur);                                                        \
+			int cmp = comparator(node, cur_node);                  \
-                                                                                                               \
+                                                                               \
-				if (!next) {                                                                   \
+			if (cmp == 1) {                                        \
-					btree_put_left(cur, &node->container_node_member);                     \
+				next = btree_right(cur);                       \
-					break;                                                                 \
+                                                                               \
-				}                                                                              \
+				if (!next) {                                   \
-			} else {                                                                               \
+					btree_put_right(                       \
-				return;                                                                        \
+						cur,                           \
-			}                                                                                      \
+						&node->container_node_member); \
-                                                                                                               \
+					break;                                 \
-			cur = next;                                                                            \
+				}                                              \
-		}                                                                                              \
+			} else if (cmp == -1) {                                \
-                                                                                                               \
+				next = btree_left(cur);                        \
-		btree_insert_fixup(tree, &node->container_node_member);                                        \
+                                                                               \
+				if (!next) {                                   \
+					btree_put_left(                        \
+						cur,                           \
+						&node->container_node_member); \
+					break;                                 \
+				}                                              \
+			} else {                                               \
+				return;                                        \
+			}                                                      \
+                                                                               \
+			cur = next;                                            \
+		}                                                              \
+                                                                               \
+		btree_insert_fixup(tree, &node->container_node_member);        \
 	}
 
 /* defines a simple tree search function.
-   this function assumes that your nodes have simple integer keys that can be compared with the usual operators.
+   this function assumes that your nodes have simple integer keys that can be
+   compared with the usual operators.
 
    EXAMPLE:
    if you have a tree node type like this:
 
      struct my_tree_node {
              int key;
-	     struct btree_node base;
+             struct btree_node base;
      }
 
-   You would use the following call to generate a search function for a tree with this node type:
+   You would use the following call to generate a search function for a tree
+   with this node type:
 
-     BTREE_DEFINE_SIMPLE_GET(struct my_tree_node, int, base, key, my_tree_node_get);
+     BTREE_DEFINE_SIMPLE_GET(struct my_tree_node, int, base, key,
+   my_tree_node_get);
 
    Which would emit a function defined like:
 
      static struct my_tree_node *my_tree_node_get(struct btree *tree, int key);
 
-   @param node_type your custom tree node type. usually a structure that contains a struct btree_node member.
+   @param node_type your custom tree node type. usually a structure that
-   @param key_type the type name of the key embedded in your custom tree node type. this type must be
+   contains a struct btree_node member.
-                   compatible with the builtin comparison operators.
+   @param key_type the type name of the key embedded in your custom tree node
-   @param container_node_member the name of the struct btree_node member variable within your custom type.
+   type. this type must be compatible with the builtin comparison operators.
-   @param container_key_member the name of the key member variable within your custom type.
+   @param container_node_member the name of the struct btree_node member
+   variable within your custom type.
+   @param container_key_member the name of the key member variable within your
+   custom type.
    @param function_name the name of the function to generate.
 */
-#define BTREE_DEFINE_SIMPLE_GET(node_type, key_type, container_node_member, container_key_member, function_name) \
+#define BTREE_DEFINE_SIMPLE_GET(                                               \
-	node_type *function_name(struct btree *tree, key_type key)                                               \
+	node_type,                                                             \
-	{                                                                                                        \
+	key_type,                                                              \
-		struct btree_node *cur = tree->b_root;                                                           \
+	container_node_member,                                                 \
-		while (cur) {                                                                                    \
+	container_key_member,                                                  \
-			node_type *cur_node = BTREE_CONTAINER(node_type, container_node_member, cur);            \
+	function_name)                                                         \
-			if (key > cur_node->container_key_member) {                                              \
+	node_type *function_name(struct btree *tree, key_type key)             \
-				cur = btree_right(cur);                                                          \
+	{                                                                      \
-			} else if (key < cur_node->container_key_member) {                                       \
+		struct btree_node *cur = tree->b_root;                         \
-				cur = btree_left(cur);                                                           \
+		while (cur) {                                                  \
-			} else {                                                                                 \
+			node_type *cur_node = BTREE_CONTAINER(                 \
-				return cur_node;                                                                 \
+				node_type,                                     \
-			}                                                                                        \
+				container_node_member,                         \
-		}                                                                                                \
+				cur);                                          \
-	                                                                                                         \
+			if (key > cur_node->container_key_member) {            \
-		return NULL;                                                                                     \
+				cur = btree_right(cur);                        \
+			} else if (key < cur_node->container_key_member) {     \
+				cur = btree_left(cur);                         \
+			} else {                                               \
+				return cur_node;                               \
+			}                                                      \
+		}                                                              \
+                                                                               \
+		return NULL;                                                   \
 	}
 
 /* perform an in-order traversal of a binary tree
@@ -230,7 +295,7 @@ extern "C" {
 
      struct my_tree_node {
              int key;
-	     struct btree_node base;
+             struct btree_node base;
      }
 
    and you want to do something like:
@@ -241,15 +306,23 @@ extern "C" {
 
      btree_foreach (struct my_tree_node, node, &my_tree, base) { ... }
 
-   @param iter_type the type name of the iterator variable. this should be the tree's node type, and shouldn't be a pointer.
+   @param iter_type the type name of the iterator variable. this should be the
+   tree's node type, and shouldn't be a pointer.
    @param iter_name the name of the iterator variable.
    @param tree_name a pointer to the tree to traverse.
-   @param node_member the name of the struct btree_node member variable within the tree node type.
+   @param node_member the name of the struct btree_node member variable within
+   the tree node type.
 */
-#define btree_foreach(iter_type, iter_name, tree_name, node_member) \
+#define btree_foreach(iter_type, iter_name, tree_name, node_member)            \
-	for (iter_type *iter_name = BTREE_CONTAINER(iter_type, node_member, btree_first(tree_name)); \
+	for (iter_type *iter_name = BTREE_CONTAINER(                           \
-	     iter_name; \
+		     iter_type,                                                \
-	     iter_name = BTREE_CONTAINER(iter_type, node_member, btree_next(&((iter_name)->node_member))))
+		     node_member,                                              \
+		     btree_first(tree_name));                                  \
+	     iter_name;                                                        \
+	     iter_name = BTREE_CONTAINER(                                      \
+		     iter_type,                                                \
+		     node_member,                                              \
+		     btree_next(&((iter_name)->node_member))))
 
 /* perform an reverse in-order traversal of a binary tree
 
@@ -261,7 +334,7 @@ extern "C" {
 
      struct my_tree_node {
              int key;
-	     struct btree_node base;
+             struct btree_node base;
      }
 
    and you want to do something like:
@@ -272,35 +345,43 @@ extern "C" {
 
      btree_foreach_r (struct my_tree_node, node, &my_tree, base) { ... }
 
-   @param iter_type the type name of the iterator variable. this should be the tree's node type, and shouldn't be a pointer.
+   @param iter_type the type name of the iterator variable. this should be the
+   tree's node type, and shouldn't be a pointer.
    @param iter_name the name of the iterator variable.
    @param tree_name a pointer to the tree to traverse.
-   @param node_member the name of the struct btree_node member variable within the tree node type.
+   @param node_member the name of the struct btree_node member variable within
+   the tree node type.
 */
-#define btree_foreach_r(iter_type, iter_name, tree_name, node_member) \
+#define btree_foreach_r(iter_type, iter_name, tree_name, node_member)          \
-	for (iter_type *iter_name = BTREE_CONTAINER(iter_type, node_member, btree_last(tree_name)); \
+	for (iter_type *iter_name                                              \
-	     iter_name; \
+	     = BTREE_CONTAINER(iter_type, node_member, btree_last(tree_name)); \
-	     iter_name = BTREE_CONTAINER(iter_type, node_member, btree_prev(&((iter_name)->node_member))))
+	     iter_name;                                                        \
+	     iter_name = BTREE_CONTAINER(                                      \
+		     iter_type,                                                \
+		     node_member,                                              \
+		     btree_prev(&((iter_name)->node_member))))
 
-/* binary tree nodes. this *cannot* be used directly. you need to define a custom node type
+/* binary tree nodes. this *cannot* be used directly. you need to define a
-   that contains a member variable of type struct btree_node.
+   custom node type that contains a member variable of type struct btree_node.
 
-   you would then use the supplied macros to define functions to manipulate your custom binary tree.
+   you would then use the supplied macros to define functions to manipulate your
+   custom binary tree.
 */
 struct btree_node {
 	struct btree_node *b_parent, *b_left, *b_right;
 	unsigned short b_height;
 };
 
-/* binary tree. unlike struct btree_node, you can define variables of type struct btree. */
+/* binary tree. unlike struct btree_node, you can define variables of type
+ * struct btree. */
 struct btree {
 	struct btree_node *b_root;
 };
 
 /* re-balance a binary tree after an insertion operation.
 
-   NOTE that, if you define an insertion function using BTREE_DEFINE_INSERT or similar,
+   NOTE that, if you define an insertion function using BTREE_DEFINE_INSERT or
-   this function will automatically called for you.
+   similar, this function will automatically called for you.
 
    @param tree the tree to re-balance.
    @param node the node that was just inserted into the tree.
@@ -316,29 +397,42 @@ extern void btree_delete(struct btree *tree, struct btree_node *node);
 
 /* get the first node in a binary tree.
 
-   this will be the node with the smallest key (i.e. the node that is furthest-left from the root)
+   this will be the node with the smallest key (i.e. the node that is
+   furthest-left from the root)
 */
 extern struct btree_node *btree_first(struct btree *tree);
 
 /* get the last node in a binary tree.
 
-   this will be the node with the largest key (i.e. the node that is furthest-right from the root)
+   this will be the node with the largest key (i.e. the node that is
+   furthest-right from the root)
 */
 extern struct btree_node *btree_last(struct btree *tree);
-/* for any binary tree node, this function returns the node with the next-largest key value */
+/* for any binary tree node, this function returns the node with the
+ * next-largest key value */
 extern struct btree_node *btree_next(struct btree_node *node);
-/* for any binary tree node, this function returns the node with the next-smallest key value */
+/* for any binary tree node, this function returns the node with the
+ * next-smallest key value */
 extern struct btree_node *btree_prev(struct btree_node *node);
 
+static inline bool btree_empty(const struct btree *tree)
+{
+	return tree->b_root == NULL;
+}
+
 /* sets `child` as the immediate left-child of `parent` */
-static inline void btree_put_left(struct btree_node *parent, struct btree_node *child)
+static inline void btree_put_left(
+	struct btree_node *parent,
+	struct btree_node *child)
 {
 	parent->b_left = child;
 	child->b_parent = parent;
 }
 
 /* sets `child` as the immediate right-child of `parent` */
-static inline void btree_put_right(struct btree_node *parent, struct btree_node *child)
+static inline void btree_put_right(
+	struct btree_node *parent,
+	struct btree_node *child)
 {
 	parent->b_right = child;
 	child->b_parent = parent;

include/kernel/channel.h

@@ -0,0 +1,55 @@
+#ifndef KERNEL_CHANNEL_H_
+#define KERNEL_CHANNEL_H_
+
+#include <kernel/object.h>
+#include <kernel/sched.h>
+
+struct msg;
+
+struct channel {
+	struct object c_base;
+	unsigned int c_id;
+	struct waitqueue c_wq;
+	struct btree c_msg;
+	struct btree_node c_node;
+};
+
+extern kern_status_t channel_type_init(void);
+extern struct channel *channel_cast(struct object *obj);
+
+extern struct channel *channel_create(void);
+
+extern kern_status_t channel_enqueue_msg(
+	struct channel *channel,
+	struct msg *msg);
+
+extern kern_status_t channel_recv_msg(
+	struct channel *channel,
+	kern_msg_t *out_msg,
+	unsigned long *irq_flags);
+extern kern_status_t channel_reply_msg(
+	struct channel *channel,
+	msgid_t id,
+	const kern_msg_t *reply,
+	unsigned long *irq_flags);
+
+extern kern_status_t channel_read_msg(
+	struct channel *channel,
+	msgid_t msg,
+	size_t offset,
+	struct vm_region *dest_region,
+	const kern_iovec_t *dest_iov,
+	size_t dest_iov_count,
+	size_t *nr_read);
+extern kern_status_t channel_write_msg(
+	struct channel *channel,
+	msgid_t msg,
+	size_t offset,
+	struct vm_region *src_region,
+	const kern_iovec_t *src_iov,
+	size_t src_iov_count,
+	size_t *nr_written);
+
+DEFINE_OBJECT_LOCK_FUNCTION(channel, c_base)
+
+#endif

include/kernel/clock.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_CLOCK_H_
+#ifndef KERNEL_CLOCK_H_
-#define MANGO_CLOCK_H_
+#define KERNEL_CLOCK_H_
 
 #include <stdint.h>
 

include/kernel/compiler.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_COMPILER_H_
+#ifndef KERNEL_COMPILER_H_
-#define MANGO_COMPILER_H_
+#define KERNEL_COMPILER_H_
 
 #ifdef __cplusplus
 template <typename T>

include/kernel/console.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_CONSOLE_H_
+#ifndef KERNEL_CONSOLE_H_
-#define MANGO_CONSOLE_H_
+#define KERNEL_CONSOLE_H_
 
 /* The console system
 
@@ -14,9 +14,10 @@
    representing a serial port may allow both sending AND receiving over the
    port.
 */
-#include <mango/queue.h>
+#include <kernel/locks.h>
-#include <mango/locks.h>
+#include <kernel/queue.h>
 #include <mango/status.h>
+#include <mango/types.h>
 
 #ifdef __cplusplus
 extern "C" {

include/kernel/cpu.h

@@ -1,10 +1,10 @@
-#ifndef MANGO_CPU_H_
+#ifndef KERNEL_CPU_H_
-#define MANGO_CPU_H_
+#define KERNEL_CPU_H_
 
-#include <mango/types.h>
+#include <kernel/types.h>
-#include <mango/machine/cpu.h>
+#include <kernel/machine/cpu.h>
 #include <stdint.h>
-#include <mango/sched.h>
+#include <kernel/sched.h>
 
 #ifdef __cplusplus
 extern "C" {

include/kernel/fb.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_FB_H_
+#ifndef KERNEL_FB_H_
-#define MANGO_FB_H_
+#define KERNEL_FB_H_
 
 #include <stdint.h>
 

include/kernel/flags.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_FLAGS_H_
+#ifndef KERNEL_FLAGS_H_
-#define MANGO_FLAGS_H_
+#define KERNEL_FLAGS_H_
 
 #include <stdint.h>
 

include/kernel/handle.h

@@ -0,0 +1,69 @@
+#ifndef KERNEL_HANDLE_H_
+#define KERNEL_HANDLE_H_
+
+#include <kernel/bitmap.h>
+#include <mango/status.h>
+#include <mango/types.h>
+#include <stddef.h>
+#include <stdint.h>
+
+/* subtract 32 bytes to account for the handle bitmap */
+#define HANDLES_PER_TABLE ((4096 - 32) / sizeof(struct handle))
+#define REFS_PER_TABLE    ((4096 - 64) / sizeof(struct handle_table *))
+
+typedef uint32_t kern_handle_t;
+
+typedef uintptr_t handle_flags_t;
+
+struct task;
+struct object;
+struct vm_region;
+struct handle_list;
+
+struct handle {
+	struct object *h_object;
+	handle_flags_t h_flags;
+};
+
+struct handle_table {
+	union {
+		struct {
+			/* bitmap tracks which bits in t_handle_list are
+			 * allocated */
+			DECLARE_BITMAP(t_handle_map, HANDLES_PER_TABLE);
+			struct handle t_handle_list[HANDLES_PER_TABLE];
+		} t_handles;
+
+		struct {
+			/* bitmap tracks which sub-tables are fully-allocated */
+			DECLARE_BITMAP(t_subtable_map, REFS_PER_TABLE);
+			struct handle_table *t_subtable_list[REFS_PER_TABLE];
+		} t_subtables;
+	};
+};
+
+extern struct handle_table *handle_table_create(void);
+extern void handle_table_destroy(struct handle_table *tab);
+
+extern kern_status_t handle_table_alloc_handle(
+	struct handle_table *tab,
+	struct handle **out_slot,
+	kern_handle_t *out_handle);
+extern kern_status_t handle_table_free_handle(
+	struct handle_table *tab,
+	kern_handle_t handle);
+extern struct handle *handle_table_get_handle(
+	struct handle_table *tab,
+	kern_handle_t handle);
+
+extern kern_status_t handle_table_transfer(
+	struct vm_region *dst_region,
+	struct handle_table *dst,
+	kern_msg_handle_t *dst_handles,
+	size_t dst_handles_max,
+	struct vm_region *src_region,
+	struct handle_table *src,
+	kern_msg_handle_t *src_handles,
+	size_t src_handles_count);
+
+#endif

include/kernel/init.h

@@ -1,8 +1,8 @@
-#ifndef MANGO_INIT_H_
+#ifndef KERNEL_INIT_H_
-#define MANGO_INIT_H_
+#define KERNEL_INIT_H_
 
-#include <mango/compiler.h>
+#include <kernel/compiler.h>
-#include <mango/machine/init.h>
+#include <kernel/machine/init.h>
 
 #ifdef __cplusplus
 extern "C" {

include/kernel/input.h

@@ -1,8 +1,8 @@
-#ifndef MANGO_INPUT_H_
+#ifndef KERNEL_INPUT_H_
-#define MANGO_INPUT_H_
+#define KERNEL_INPUT_H_
 
 #include <stdint.h>
-#include <mango/queue.h>
+#include <kernel/queue.h>
 #include <mango/status.h>
 
 enum input_event_hook_flags {

include/kernel/iovec.h

@@ -0,0 +1,31 @@
+#ifndef KERNEL_IOVEC_H_
+#define KERNEL_IOVEC_H_
+
+#include <mango/types.h>
+#include <stddef.h>
+
+struct iovec_iterator {
+	/* if this is set, we are iterating over a list of iovecs stored in
+	 * userspace, and must go through this region to retrieve the data. */
+	struct vm_region *it_region;
+	const kern_iovec_t *it_vecs;
+	size_t it_nr_vecs;
+	size_t it_vec_ptr;
+
+	virt_addr_t it_base;
+	size_t it_len;
+};
+
+extern void iovec_iterator_begin(
+	struct iovec_iterator *it,
+	const kern_iovec_t *vecs,
+	size_t nr_vecs);
+extern void iovec_iterator_begin_user(
+	struct iovec_iterator *it,
+	struct vm_region *address_space,
+	const kern_iovec_t *vecs,
+	size_t nr_vecs);
+
+extern void iovec_iterator_seek(struct iovec_iterator *it, size_t nr_bytes);
+
+#endif

include/kernel/locks.h

@@ -0,0 +1,61 @@
+#ifndef KERNEL_LOCKS_H_
+#define KERNEL_LOCKS_H_
+
+#include <kernel/compiler.h>
+#include <kernel/machine/hwlock.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef __aligned(8) ml_hwlock_t spin_lock_t;
+
+#define SPIN_LOCK_INIT                ML_HWLOCK_INIT
+
+#define spin_lock(lck)                ml_hwlock_lock(lck);
+#define spin_unlock(lck)              ml_hwlock_unlock(lck);
+
+#define spin_lock_irq(lck)            ml_hwlock_lock_irq(lck);
+#define spin_unlock_irq(lck)          ml_hwlock_unlock_irq(lck);
+
+#define spin_lock_irqsave(lck, flags) ml_hwlock_lock_irqsave(lck, flags);
+#define spin_unlock_irqrestore(lck, flags)                                     \
+	ml_hwlock_unlock_irqrestore(lck, flags);
+
+static inline void spin_lock_pair_irqsave(
+	spin_lock_t *a,
+	spin_lock_t *b,
+	unsigned long *flags)
+{
+	if (a == b) {
+		spin_lock_irqsave(a, flags);
+	} else if (a < b) {
+		spin_lock_irqsave(a, flags);
+		spin_lock(b);
+	} else {
+		spin_lock_irqsave(b, flags);
+		spin_lock(a);
+	}
+}
+
+static inline void spin_unlock_pair_irqrestore(
+	spin_lock_t *a,
+	spin_lock_t *b,
+	unsigned long flags)
+{
+	if (a == b) {
+		spin_unlock_irqrestore(a, flags);
+	} else if (a < b) {
+		spin_unlock(b);
+		spin_unlock_irqrestore(a, flags);
+	} else {
+		spin_unlock(a);
+		spin_unlock_irqrestore(b, flags);
+	}
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

include/kernel/memblock.h

@@ -19,24 +19,25 @@
     contributors may be used to endorse or promote products derived from this
     software without specific prior written permission.
  */
-#ifndef MANGO_MEMBLOCK_H_
+#ifndef KERNEL_MEMBLOCK_H_
-#define MANGO_MEMBLOCK_H_
+#define KERNEL_MEMBLOCK_H_
 
-#include <stddef.h>
+#include <kernel/types.h>
 #include <limits.h>
-#include <mango/types.h>
+#include <stddef.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#define MEMBLOCK_INIT_MEMORY_REGION_COUNT 128
+#define MEMBLOCK_INIT_MEMORY_REGION_COUNT   128
 #define MEMBLOCK_INIT_RESERVED_REGION_COUNT 128
 
-#define __for_each_mem_range(i, type_a, type_b, p_start, p_end) \
+#define __for_each_mem_range(i, type_a, type_b, p_start, p_end)                \
-	for ((i)->__idx = 0, __next_memory_region(i, type_a, type_b, p_start, p_end); \
+	for ((i)->__idx = 0,                                                   \
-			(i)->__idx != ULLONG_MAX; \
+	    __next_memory_region(i, type_a, type_b, p_start, p_end);           \
-			__next_memory_region(i, type_a, type_b, p_start, p_end))
+	     (i)->__idx != ULLONG_MAX;                                         \
+	     __next_memory_region(i, type_a, type_b, p_start, p_end))
 
 /* iterate through all memory regions known to memblock.
 
@@ -47,7 +48,7 @@ extern "C" {
 
    @param i the iterator. this should be a pointer of type struct memblock_iter.
             for each iteration, this structure will be filled with details about
-	    the current memory region.
+            the current memory region.
    @param p_start the lower bound of the memory region to iterate through.
             if you don't want to use a lower bound, pass 0.
    @param p_end the upper bound of the memory region to iterate through.
@@ -65,7 +66,7 @@ extern "C" {
      struct memblock_iter it;
      for_each_mem_region (&it, 0x40000, 0x80000) { ... }
 */
-#define for_each_mem_range(i, p_start, p_end) \
+#define for_each_mem_range(i, p_start, p_end)                                  \
 	__for_each_mem_range(i, &memblock.memory, NULL, p_start, p_end)
 
 /* iterate through all memory regions reserved using memblock.
@@ -77,7 +78,7 @@ extern "C" {
 
    @param i the iterator. this should be a pointer of type struct memblock_iter.
             for each iteration, this structure will be filled with details about
-	    the current memory region.
+            the current memory region.
    @param p_start the lower bound of the memory region to iterate through.
             if you don't want to use a lower bound, pass 0.
    @param p_end the upper bound of the memory region to iterate through.
@@ -95,7 +96,7 @@ extern "C" {
      struct memblock_iter it;
      for_each_reserved_mem_region (&it, 0x40000, 0x80000) { ... }
 */
-#define for_each_reserved_mem_range(i, p_start, p_end) \
+#define for_each_reserved_mem_range(i, p_start, p_end)                         \
 	__for_each_mem_range(i, &memblock.reserved, NULL, p_start, p_end)
 
 /* iterate through all memory regions known by memblock to be free.
@@ -108,7 +109,7 @@ extern "C" {
 
    @param i the iterator. this should be a pointer of type struct memblock_iter.
             for each iteration, this structure will be filled with details about
-	    the current memory region.
+            the current memory region.
    @param p_start the lower bound of the memory region to iterate through.
             if you don't want to use a lower bound, pass 0.
    @param p_end the upper bound of the memory region to iterate through.
@@ -138,19 +139,25 @@ extern "C" {
      - 0x08000 -> 0x08fff
      - 0x10000 -> 0x1ffff
 */
-#define for_each_free_mem_range(i, p_start, p_end) \
+#define for_each_free_mem_range(i, p_start, p_end)                             \
-	__for_each_mem_range(i, &memblock.memory, &memblock.reserved, p_start, p_end)
+	__for_each_mem_range(                                                  \
+		i,                                                             \
+		&memblock.memory,                                              \
+		&memblock.reserved,                                            \
+		p_start,                                                       \
+		p_end)
 
 typedef uint64_t memblock_index_t;
 
 enum memblock_region_status {
-	/* Used in memblock.memory regions, indicates that the memory region exists */
+	/* Used in memblock.memory regions, indicates that the memory region
+	 * exists */
 	MEMBLOCK_MEMORY = 0,
-	/* Used in memblock.reserved regions, indicates that the memory region was reserved
+	/* Used in memblock.reserved regions, indicates that the memory region
-	 * by a call to memblock_alloc() */
+	 * was reserved by a call to memblock_alloc() */
 	MEMBLOCK_ALLOC,
-	/* Used in memblock.reserved regions, indicates that the memory region was reserved
+	/* Used in memblock.reserved regions, indicates that the memory region
-	 * by a call to memblock_reserve() */
+	 * was reserved by a call to memblock_reserve() */
 	MEMBLOCK_RESERVED,
 };
 
@@ -176,9 +183,10 @@ struct memblock {
 	/* bounds of the memory region that can be used by memblock_alloc()
 	   both of these are virtual addresses */
 	uintptr_t m_alloc_start, m_alloc_end;
-	/* memblock assumes that all memory in the alloc zone is contiguously mapped
+	/* memblock assumes that all memory in the alloc zone is contiguously
-	   (if paging is enabled). m_voffset is the offset that needs to be added to
+	   mapped (if paging is enabled). m_voffset is the offset that needs to
-	   a given physical address to get the corresponding virtual address */
+	   be added to a given physical address to get the corresponding virtual
+	   address */
 	uintptr_t m_voffset;
 
 	struct memblock_type memory;
@@ -212,7 +220,10 @@ extern int __next_mem_range(struct memblock_iter *it);
    @param voffset the offset between the physical address of a given page and
                   its corresponding virtual address.
 */
-extern int memblock_init(uintptr_t alloc_start, uintptr_t alloc_end, uintptr_t voffset);
+extern int memblock_init(
+	uintptr_t alloc_start,
+	uintptr_t alloc_end,
+	uintptr_t voffset);
 
 /* add a region of memory to memblock.
 
@@ -234,7 +245,8 @@ extern int memblock_add(phys_addr_t base, size_t size);
    reserved memory will not be used by memblock_alloc(), and will remain
    reserved when the vm_page memory map is initialised.
 
-   @param base the physical address of the start of the memory region to reserve.
+   @param base the physical address of the start of the memory region to
+   reserve.
    @oaram size the size of the memory region to reserve in bytes.
 */
 extern int memblock_reserve(phys_addr_t base, size_t size);
@@ -257,7 +269,7 @@ extern int memblock_reserve(phys_addr_t base, size_t size);
    @param size the size of the buffer to allocate in bytes.
    @param align the alignment to use. for example, an alignment of 4096
                 will result in the returned pointer being a multiple
-		of 4096. this must be a power of 2.
+                of 4096. this must be a power of 2.
 */
 extern void *memblock_alloc(size_t size, phys_addr_t align);
 
@@ -279,7 +291,7 @@ extern void *memblock_alloc(size_t size, phys_addr_t align);
    @param size the size of the buffer to allocate in bytes.
    @param align the alignment to use. for example, an alignment of 4096
                 will result in the returned pointer being a multiple
-		of 4096. this must be a power of 2.
+                of 4096. this must be a power of 2.
 */
 extern phys_addr_t memblock_alloc_phys(size_t size, phys_addr_t align);
 
@@ -310,7 +322,7 @@ extern int memblock_free_phys(phys_addr_t addr, size_t size);
 
    @param p the pointer to convert.
 */
-extern phys_addr_t memblock_virt_to_phys(void *p);
+extern phys_addr_t memblock_virt_to_phys(const void *p);
 
 /* convert a physical address returned by memblock
    to a virtual pointer.
@@ -319,9 +331,14 @@ extern phys_addr_t memblock_virt_to_phys(void *p);
 */
 extern void *memblock_phys_to_virt(phys_addr_t p);
 
-extern void __next_memory_region(struct memblock_iter *it, \
+extern void __next_memory_region(
-		struct memblock_type *type_a, struct memblock_type *type_b,
+	struct memblock_iter *it,
-		phys_addr_t start, phys_addr_t end);
+	struct memblock_type *type_a,
+	struct memblock_type *type_b,
+	phys_addr_t start,
+	phys_addr_t end);
+
+extern void memblock_dump(void);
 
 #ifdef __cplusplus
 }

include/kernel/msg.h

@@ -0,0 +1,29 @@
+#ifndef KERNEL_MSG_H_
+#define KERNEL_MSG_H_
+
+#include <kernel/btree.h>
+#include <kernel/locks.h>
+#include <mango/status.h>
+#include <mango/types.h>
+
+struct port;
+struct thread;
+
+enum kmsg_status {
+	KMSG_WAIT_RECEIVE,
+	KMSG_WAIT_REPLY,
+	KMSG_REPLY_SENT,
+};
+
+struct msg {
+	spin_lock_t msg_lock;
+	enum kmsg_status msg_status;
+	struct btree_node msg_node;
+	msgid_t msg_id;
+	kern_status_t msg_result;
+	struct port *msg_sender_port;
+	struct thread *msg_sender_thread;
+	kern_msg_t msg_req, msg_resp;
+};
+
+#endif

include/kernel/object.h

@@ -0,0 +1,122 @@
+#ifndef KERNEL_OBJECT_H_
+#define KERNEL_OBJECT_H_
+
+#include <kernel/flags.h>
+#include <kernel/locks.h>
+#include <kernel/vm.h>
+#include <mango/status.h>
+#include <stddef.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define DEFINE_OBJECT_LOCK_FUNCTION(object_name, base)                         \
+	static inline void object_name##_lock(struct object_name *p)           \
+	{                                                                      \
+		object_lock(&p->base);                                         \
+	}                                                                      \
+	static inline void object_name##_unlock(struct object_name *p)         \
+	{                                                                      \
+		object_unlock(&p->base);                                       \
+	}                                                                      \
+	static inline void object_name##_lock_irqsave(                         \
+		struct object_name *p,                                         \
+		unsigned long *flags)                                          \
+	{                                                                      \
+		object_lock_irqsave(&p->base, flags);                          \
+	}                                                                      \
+	static inline void object_name##_unlock_irqrestore(                    \
+		struct object_name *p,                                         \
+		unsigned long flags)                                           \
+	{                                                                      \
+		object_unlock_irqrestore(&p->base, flags);                     \
+	}                                                                      \
+	static inline void object_name##_lock_pair_irqsave(                    \
+		struct object_name *a,                                         \
+		struct object_name *b,                                         \
+		unsigned long *flags)                                          \
+	{                                                                      \
+		object_lock_pair_irqsave(&a->base, &b->base, flags);           \
+	}                                                                      \
+	static inline void object_name##_unlock_pair_irqrestore(               \
+		struct object_name *a,                                         \
+		struct object_name *b,                                         \
+		unsigned long flags)                                           \
+	{                                                                      \
+		object_unlock_pair_irqrestore(&a->base, &b->base, flags);      \
+	}
+
+#define OBJECT_MAGIC    0xBADDCAFE
+#define OBJECT_NAME_MAX 64
+#define OBJECT_PATH_MAX 256
+
+#define OBJECT_CAST(to_type, to_type_member, p)                                \
+	((to_type *)((uintptr_t)p) - offsetof(to_type, to_type_member))
+#define OBJECT_C_CAST(c_type, c_type_member, obj_type, objp)                   \
+	OBJECT_IS_TYPE(objp, obj_type)                                         \
+	? OBJECT_CAST(c_type, c_type_member, (objp)) : NULL
+#define OBJECT_IS_TYPE(obj, type_ptr) ((obj)->ob_type == (type_ptr))
+
+struct object;
+struct object_attrib;
+
+enum object_type_flags {
+	OBJTYPE_INIT = 0x01u,
+};
+
+struct object_ops {
+	kern_status_t (*destroy)(struct object *obj, struct queue *q);
+	kern_status_t (*destroy_recurse)(
+		struct queue_entry *entry,
+		struct object **out);
+};
+
+struct object_type {
+	enum object_type_flags ob_flags;
+	char ob_name[32];
+	unsigned int ob_size;
+	unsigned int ob_header_offset;
+	struct vm_cache ob_cache;
+	struct queue_entry ob_list;
+	struct object_ops ob_ops;
+};
+
+struct object {
+	uint32_t ob_magic;
+	koid_t ob_id;
+	struct object_type *ob_type;
+	spin_lock_t ob_lock;
+	unsigned int ob_refcount;
+	unsigned int ob_handles;
+	struct queue_entry ob_list;
+} __aligned(sizeof(long));
+
+extern kern_status_t object_bootstrap(void);
+extern kern_status_t object_type_register(struct object_type *p);
+extern kern_status_t object_type_unregister(struct object_type *p);
+
+extern struct object *object_create(struct object_type *type);
+extern struct object *object_ref(struct object *obj);
+extern void object_unref(struct object *obj);
+extern void object_add_handle(struct object *obj);
+extern void object_remove_handle(struct object *obj);
+extern void object_lock(struct object *obj);
+extern void object_unlock(struct object *obj);
+extern void object_lock_irqsave(struct object *obj, unsigned long *flags);
+extern void object_unlock_irqrestore(struct object *obj, unsigned long flags);
+
+extern void object_lock_pair_irqsave(
+	struct object *a,
+	struct object *b,
+	unsigned long *flags);
+extern void object_unlock_pair_irqrestore(
+	struct object *a,
+	struct object *b,
+	unsigned long flags);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

include/kernel/panic.h

@@ -0,0 +1,13 @@
+#ifndef KERNEL_PANIC_H_
+#define KERNEL_PANIC_H_
+
+#include <kernel/compiler.h>
+
+struct ml_cpu_context;
+
+#define panic(...) panic_irq(NULL, __VA_ARGS__)
+
+extern void __noreturn
+panic_irq(struct ml_cpu_context *ctx, const char *fmt, ...);
+
+#endif

include/kernel/percpu.h

@@ -1,9 +1,9 @@
-#ifndef MANGO_PERCPU_H_
+#ifndef KERNEL_PERCPU_H_
-#define MANGO_PERCPU_H_
+#define KERNEL_PERCPU_H_
 
 #include <mango/status.h>
-#include <mango/compiler.h>
+#include <kernel/compiler.h>
-#include <mango/sched.h>
+#include <kernel/sched.h>
 
 #ifdef __cplusplus
 extern "C" {

include/kernel/pmap.h

@@ -0,0 +1,81 @@
+#ifndef KERNEL_PMAP_H_
+#define KERNEL_PMAP_H_
+
+/* all the functions declared in this file are defined in arch/xyz/pmap.c */
+
+#include <kernel/machine/pmap.h>
+#include <kernel/vm.h>
+#include <mango/status.h>
+#include <stddef.h>
+
+#define PMAP_INVALID ML_PMAP_INVALID
+#define PFN(x)       ((x) >> VM_PAGE_SHIFT)
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef ml_pmap_t pmap_t;
+typedef ml_pfn_t pfn_t;
+
+enum pmap_fault_flags {
+	/* if set, the faulting page is present, and the fault is
+	 * protection-related.
+	 * if clear, the faulting page is missing, and the
+	 * fault is due to the missing page.
+	 */
+	PMAP_FAULT_PRESENT = 0x01u,
+	/* if set, the faulting page was accessed from user mode.
+	 * if clear, the faulting page was accessed from kernel mode.
+	 */
+	PMAP_FAULT_USER = 0x02u,
+	/* if set, the fault was caused by a write operation.
+	 * if clear, the faulting page was caused by a read operation.
+	 */
+	PMAP_FAULT_WRITE = 0x04u,
+	/* if set, the fault was caused while fetching an instruction from the
+	 * faulting page.
+	 */
+	PMAP_FAULT_IFETCH = 0x08u,
+	/* if set, the fault was caused by misconfigured page tables */
+	PMAP_FAULT_BADCFG = 0x10u,
+};
+
+enum pmap_flags {
+	PMAP_NORMAL = 0x00u,
+	PMAP_HUGEPAGE = 0x01u,
+};
+
+extern void pmap_bootstrap(void);
+extern pmap_t get_kernel_pmap(void);
+
+extern pmap_t pmap_create(void);
+extern void pmap_destroy(pmap_t pmap);
+extern void pmap_switch(pmap_t pmap);
+
+extern kern_status_t pmap_handle_fault(
+	virt_addr_t fault_addr,
+	enum pmap_fault_flags flags);
+
+extern kern_status_t pmap_add(
+	pmap_t pmap,
+	virt_addr_t p,
+	pfn_t pfn,
+	vm_prot_t prot,
+	enum pmap_flags flags);
+extern kern_status_t pmap_add_block(
+	pmap_t pmap,
+	virt_addr_t p,
+	pfn_t pfn,
+	size_t len,
+	vm_prot_t prot,
+	enum pmap_flags flags);
+
+extern kern_status_t pmap_remove(pmap_t pmap, virt_addr_t p);
+extern kern_status_t pmap_remove_range(pmap_t pmap, virt_addr_t p, size_t len);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

include/kernel/port.h

@@ -0,0 +1,41 @@
+#ifndef KERNEL_PORT_H_
+#define KERNEL_PORT_H_
+
+#include <kernel/object.h>
+#include <kernel/sched.h>
+
+enum port_status {
+	/* port is not connected */
+	PORT_OFFLINE = 0,
+	/* port is connected and ready to send messages */
+	PORT_READY,
+	/* port has sent a message, and is waiting for the remote to receive it
+	 */
+	PORT_SEND_BLOCKED,
+	/* port has sent a message, and the remote has received it. waiting for
+	 * the remote to reply. */
+	PORT_REPLY_BLOCKED,
+};
+
+struct port {
+	struct object p_base;
+	enum port_status p_status;
+	struct channel *p_remote;
+};
+
+extern kern_status_t port_type_init(void);
+extern struct port *port_cast(struct object *obj);
+
+extern struct port *port_create(void);
+
+extern kern_status_t port_connect(struct port *port, struct channel *remote);
+extern kern_status_t port_disconnect(struct port *port);
+extern kern_status_t port_send_msg(
+	struct port *port,
+	const kern_msg_t *msg,
+	kern_msg_t *out_response,
+	unsigned long *lock_flags);
+
+DEFINE_OBJECT_LOCK_FUNCTION(port, p_base)
+
+#endif

include/kernel/printk.h

@@ -0,0 +1,25 @@
+#ifndef KERNEL_PRINTK_H_
+#define KERNEL_PRINTK_H_
+
+#include <kernel/console.h>
+
+#undef TRACE
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef TRACE
+#define tracek(...) printk(__VA_ARGS__)
+#else
+#define tracek(...)
+#endif
+
+extern void early_printk_init(struct console *con);
+extern int printk(const char *format, ...);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

include/kernel/queue.h

@@ -1,7 +1,7 @@
-#ifndef MANGO_QUEUE_H_
+#ifndef KERNEL_QUEUE_H_
-#define MANGO_QUEUE_H_
+#define KERNEL_QUEUE_H_
 
-#include <mango/libc/string.h>
+#include <kernel/libc/string.h>
 #include <stdbool.h>
 
 #ifdef __cplusplus

include/kernel/ringbuffer.h

@@ -1,8 +1,8 @@
-#ifndef MANGO_RINGBUFFER_H_
+#ifndef KERNEL_RINGBUFFER_H_
-#define MANGO_RINGBUFFER_H_
+#define KERNEL_RINGBUFFER_H_
 
-#include <mango/locks.h>
+#include <kernel/locks.h>
-#include <mango/sched.h>
+#include <kernel/sched.h>
 
 struct ringbuffer {
 	unsigned char *r_buffer;

include/kernel/sched.h

@@ -1,15 +1,18 @@
-#ifndef MANGO_SCHED_H_
+#ifndef KERNEL_SCHED_H_
-#define MANGO_SCHED_H_
+#define KERNEL_SCHED_H_
 
-#include <mango/btree.h>
+#include <kernel/btree.h>
-#include <mango/locks.h>
+#include <kernel/handle.h>
-#include <mango/object.h>
+#include <kernel/locks.h>
-#include <mango/pmap.h>
+#include <kernel/msg.h>
-#include <mango/queue.h>
+#include <kernel/object.h>
+#include <kernel/pmap.h>
+#include <kernel/queue.h>
 #include <mango/status.h>
 
 #define TASK_NAME_MAX       64
 #define PRIO_MAX            32
+#define PID_MAX             99999
 #define THREAD_KSTACK_ORDER VM_PAGE_4K
 #define THREAD_MAX          65536
 
@@ -32,6 +35,7 @@
 extern "C" {
 #endif
 
+struct channel;
 struct runqueue;
 struct work_item;
 
@@ -75,13 +79,20 @@ struct task {
 	struct object t_base;
 
 	struct task *t_parent;
-	unsigned int t_id;
+	long t_id;
 	enum task_state t_state;
 	char t_name[TASK_NAME_MAX];
 
 	pmap_t t_pmap;
+	struct vm_region *t_address_space;
+	spin_lock_t t_handles_lock;
+	struct handle_table *t_handles;
+	struct btree b_channels;
 
 	struct btree_node t_tasklist;
+	struct queue_entry t_child_entry;
+
+	size_t t_next_thread_id;
 	struct queue t_threads;
 	struct queue t_children;
 };
@@ -100,14 +111,17 @@ struct thread {
 	cycles_t tr_quantum_cycles, tr_quantum_target;
 	cycles_t tr_total_cycles;
 
-	uintptr_t tr_sp, tr_bp;
+	virt_addr_t tr_ip, tr_sp, tr_bp;
+	virt_addr_t tr_cpu_user_sp, tr_cpu_kernel_sp;
 
 	struct runqueue *tr_rq;
+	struct msg tr_msg;
 
-	struct queue_entry tr_threads;
+	struct queue_entry tr_parent_entry;
 	struct queue_entry tr_rqentry;
 
 	struct vm_page *tr_kstack;
+	struct vm_object *tr_ustack;
 };
 
 struct runqueue {
@@ -175,15 +189,35 @@ extern void rq_remove_thread(struct runqueue *rq, struct thread *thr);
 extern struct runqueue *cpu_rq(unsigned int cpu);
 
 extern struct task *task_alloc(void);
+extern struct task *task_cast(struct object *obj);
+extern struct task *task_create(const char *name, size_t name_len);
 static inline struct task *task_ref(struct task *task)
 {
 	return OBJECT_CAST(struct task, t_base, object_ref(&task->t_base));
 }
-static inline void task_deref(struct task *task)
+static inline void task_unref(struct task *task)
 {
-	object_deref(&task->t_base);
+	object_unref(&task->t_base);
 }
-extern struct task *task_from_pid(unsigned int pid);
+extern kern_status_t task_add_child(struct task *parent, struct task *child);
+extern kern_status_t task_add_channel(
+	struct task *task,
+	struct channel *channel,
+	unsigned int id);
+extern struct channel *task_get_channel(struct task *task, unsigned int id);
+extern struct task *task_from_tid(tid_t id);
+extern kern_status_t task_open_handle(
+	struct task *task,
+	struct object *obj,
+	handle_flags_t flags,
+	kern_handle_t *out);
+extern kern_status_t task_resolve_handle(
+	struct task *task,
+	kern_handle_t handle,
+	struct object **out_obj,
+	handle_flags_t *out_flags);
+extern kern_status_t task_close_handle(struct task *task, kern_handle_t handle);
+extern struct thread *task_create_thread(struct task *parent);
 extern struct task *kernel_task(void);
 extern struct task *idle_task(void);
 extern cycles_t default_quantum(void);
@@ -198,21 +232,19 @@ extern void schedule_thread_on_cpu(struct thread *thr);
 extern void start_charge_period(void);
 extern void end_charge_period(void);
 
-static inline void task_lock_irqsave(struct task *task, unsigned long *flags)
+DEFINE_OBJECT_LOCK_FUNCTION(task, t_base)
-{
-	object_lock_irqsave(&task->t_base, flags);
-}
-
-static inline void task_unlock_irqrestore(
-	struct task *task,
-	unsigned long flags)
-{
-	object_unlock_irqrestore(&task->t_base, flags);
-}
 
 extern struct thread *thread_alloc(void);
-extern kern_status_t thread_init(struct thread *thr, uintptr_t ip);
+extern struct thread *thread_cast(struct object *obj);
+extern kern_status_t thread_init_kernel(struct thread *thr, virt_addr_t ip);
+extern kern_status_t thread_init_user(
+	struct thread *thr,
+	virt_addr_t ip,
+	virt_addr_t sp,
+	const uintptr_t *args,
+	size_t nr_args);
 extern int thread_priority(struct thread *thr);
+extern void thread_awaken(struct thread *thr);
 extern void idle(void);
 extern struct thread *create_kernel_thread(void (*fn)(void));
 extern struct thread *create_idle_thread(void);

include/kernel/syscall.h

@@ -0,0 +1,181 @@
+#ifndef KERNEL_SYSCALL_H_
+#define KERNEL_SYSCALL_H_
+
+#include <kernel/handle.h>
+#include <kernel/sched.h>
+#include <kernel/vm-region.h>
+#include <kernel/vm.h>
+#include <mango/status.h>
+#include <mango/syscall.h>
+
+#define validate_access(task, ptr, len, flags)                                 \
+	__validate_access(task, (const void *)ptr, len, flags)
+#define validate_access_r(task, ptr, len)                                      \
+	validate_access(task, ptr, len, VM_PROT_READ | VM_PROT_USER)
+#define validate_access_w(task, ptr, len)                                      \
+	validate_access(task, ptr, len, VM_PROT_WRITE | VM_PROT_USER)
+#define validate_access_rw(task, ptr, len)                                     \
+	validate_access(                                                       \
+		task,                                                          \
+		ptr,                                                           \
+		len,                                                           \
+		VM_PROT_READ | VM_PROT_WRITE | VM_PROT_USER)
+
+static inline bool __validate_access(
+	struct task *task,
+	const void *ptr,
+	size_t len,
+	vm_prot_t flags)
+{
+	unsigned long irq_flags;
+	vm_region_lock_irqsave(task->t_address_space, &irq_flags);
+	bool result = vm_region_validate_access(
+		task->t_address_space,
+		(virt_addr_t)ptr,
+		len,
+		flags | VM_PROT_USER);
+	vm_region_unlock_irqrestore(task->t_address_space, irq_flags);
+	return result;
+}
+
+extern kern_status_t sys_task_exit(int status);
+extern kern_status_t sys_task_self(kern_handle_t *out);
+extern kern_status_t sys_task_create(
+	kern_handle_t parent_handle,
+	const char *name,
+	size_t name_len,
+	kern_handle_t *out_task,
+	kern_handle_t *out_address_space);
+extern kern_status_t sys_task_create_thread(
+	kern_handle_t task,
+	virt_addr_t ip,
+	virt_addr_t sp,
+	uintptr_t *args,
+	size_t nr_args,
+	kern_handle_t *out_thread);
+extern kern_status_t sys_task_get_address_space(
+	kern_handle_t task,
+	kern_handle_t *out);
+
+extern kern_status_t sys_thread_start(kern_handle_t thread);
+
+extern kern_status_t sys_vm_object_create(
+	const char *name,
+	size_t name_len,
+	size_t data_len,
+	vm_prot_t prot,
+	kern_handle_t *out);
+extern kern_status_t sys_vm_object_read(
+	kern_handle_t object,
+	void *dst,
+	off_t offset,
+	size_t count,
+	size_t *nr_read);
+extern kern_status_t sys_vm_object_write(
+	kern_handle_t object,
+	const void *src,
+	off_t offset,
+	size_t count,
+	size_t *nr_written);
+extern kern_status_t sys_vm_object_copy(
+	kern_handle_t dst,
+	off_t dst_offset,
+	kern_handle_t src,
+	off_t src_offset,
+	size_t count,
+	size_t *nr_copied);
+
+extern kern_status_t sys_vm_region_create(
+	kern_handle_t parent,
+	const char *name,
+	size_t name_len,
+	off_t offset,
+	size_t region_len,
+	vm_prot_t prot,
+	kern_handle_t *out,
+	virt_addr_t *out_base_address);
+extern kern_status_t sys_vm_region_kill(kern_handle_t region);
+extern kern_status_t sys_vm_region_read(
+	kern_handle_t region,
+	void *dst,
+	off_t offset,
+	size_t count,
+	size_t *nr_read);
+extern kern_status_t sys_vm_region_write(
+	kern_handle_t region,
+	const void *src,
+	off_t offset,
+	size_t count,
+	size_t *nr_read);
+extern kern_status_t sys_vm_region_map_absolute(
+	kern_handle_t region,
+	virt_addr_t map_address,
+	kern_handle_t object,
+	off_t object_offset,
+	size_t length,
+	vm_prot_t prot,
+	virt_addr_t *out_base_address);
+extern kern_status_t sys_vm_region_map_relative(
+	kern_handle_t region,
+	off_t region_offset,
+	kern_handle_t object,
+	off_t object_offset,
+	size_t length,
+	vm_prot_t prot,
+	virt_addr_t *out_base_address);
+extern kern_status_t sys_vm_region_unmap_absolute(
+	kern_handle_t region,
+	virt_addr_t address,
+	size_t length);
+extern kern_status_t sys_vm_region_unmap_relative(
+	kern_handle_t region,
+	off_t offset,
+	size_t length);
+
+extern kern_status_t sys_kern_log(const char *s);
+extern kern_status_t sys_kern_handle_close(kern_handle_t handle);
+extern kern_status_t sys_kern_config_get(
+	kern_config_key_t key,
+	void *ptr,
+	size_t len);
+extern kern_status_t sys_kern_config_set(
+	kern_config_key_t key,
+	const void *ptr,
+	size_t len);
+
+extern kern_status_t sys_channel_create(unsigned int id, kern_handle_t *out);
+extern kern_status_t sys_port_create(kern_handle_t *out);
+extern kern_status_t sys_port_connect(
+	kern_handle_t port,
+	tid_t task_id,
+	unsigned int channel_id);
+extern kern_status_t sys_port_disconnect(kern_handle_t port);
+
+extern kern_status_t sys_msg_send(
+	kern_handle_t port,
+	const kern_msg_t *msg,
+	kern_msg_t *out_reply);
+extern kern_status_t sys_msg_recv(kern_handle_t channel, kern_msg_t *out_msg);
+
+extern kern_status_t sys_msg_reply(
+	kern_handle_t channel,
+	msgid_t id,
+	const kern_msg_t *msg);
+extern kern_status_t sys_msg_read(
+	kern_handle_t channel_handle,
+	msgid_t id,
+	size_t offset,
+	const kern_iovec_t *iov,
+	size_t iov_count,
+	size_t *nr_read);
+extern kern_status_t sys_msg_write(
+	kern_handle_t channel,
+	msgid_t id,
+	size_t offset,
+	const kern_iovec_t *in,
+	size_t nr_in,
+	size_t *nr_written);
+
+extern virt_addr_t syscall_get_function(unsigned int sysid);
+
+#endif

include/kernel/test.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_TEST_H_
+#ifndef KERNEL_TEST_H_
-#define MANGO_TEST_H_
+#define KERNEL_TEST_H_
 
 #ifdef __cplusplus
 extern "C" {

include/kernel/types.h

@@ -0,0 +1,18 @@
+#ifndef KERNEL_TYPES_H_
+#define KERNEL_TYPES_H_
+
+#include <mango/types.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#define CYCLES_MAX UINT64_MAX
+
+typedef uint64_t cycles_t;
+typedef uint64_t sectors_t;
+
+struct boot_module {
+	phys_addr_t mod_base;
+	size_t mod_size;
+};
+
+#endif

include/kernel/util.h

@@ -1,27 +1,35 @@
-#ifndef MANGO_UTIL_H_
+#ifndef KERNEL_UTIL_H_
-#define MANGO_UTIL_H_
+#define KERNEL_UTIL_H_
 
+#include <mango/types.h>
+#include <stdbool.h>
 #include <stddef.h>
 #include <stdint.h>
-#include <stdbool.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#define MIN(x, y) ((x) < (y) ? (x) : (y))
+#define MIN(x, y)        ((x) < (y) ? (x) : (y))
-#define MAX(x, y) ((x) > (y) ? (x) : (y))
+#define MAX(x, y)        ((x) > (y) ? (x) : (y))
 #define CLAMP(x, lo, hi) (MIN(MAX(x, lo), hi))
 
 extern uint64_t hash_string(const char *s);
 extern void data_size_to_string(size_t value, char *out, size_t outsz);
-static inline bool power_of_2(size_t x) { return (x > 0 && (x & (x - 1)) == 0); }
+static inline bool power_of_2(size_t x)
-static inline unsigned long long div64_pow2(unsigned long long x, unsigned long long y)
+{
+	return (x > 0 && (x & (x - 1)) == 0);
+}
+static inline unsigned long long div64_pow2(
+	unsigned long long x,
+	unsigned long long y)
 {
 	return x >> (__builtin_ctz(y));
 }
 
-static inline unsigned long long absdiff64(unsigned long long x, unsigned long long y)
+static inline unsigned long long absdiff64(
+	unsigned long long x,
+	unsigned long long y)
 {
 	return x < y ? y - x : x - y;
 }
@@ -53,6 +61,8 @@ extern uint64_t host_to_little_u64(uint64_t v);
 extern uint64_t big_to_host_u64(uint64_t v);
 extern uint64_t little_to_host_u64(uint64_t v);
 
+extern bool fill_random(void *buffer, unsigned int size);
+
 #ifdef __cplusplus
 }
 #endif

include/kernel/vm-object.h

@@ -0,0 +1,91 @@
+#ifndef KERNEL_VM_OBJECT_H_
+#define KERNEL_VM_OBJECT_H_
+
+#include <kernel/locks.h>
+#include <kernel/object.h>
+
+#define VM_OBJECT_NAME_MAX 64
+
+enum vm_object_flags {
+	/* the memory behind this vm-object wasn't allocated by us, and
+	 * therefore shouldn't be freed by us */
+	VMO_IN_PLACE = 0x01u,
+};
+
+struct vm_object {
+	struct object vo_base;
+
+	char vo_name[VM_OBJECT_NAME_MAX];
+	enum vm_object_flags vo_flags;
+
+	/* queue of struct vm_region_mapping */
+	struct queue vo_mappings;
+
+	/* memory protection flags. mappings of this vm_object can only use
+	 * a subset of the flags set in this mask. */
+	vm_prot_t vo_prot;
+
+	/* btree of vm_pages that have been allocated to this vm_object.
+	 * vm_page->p_vmo_offset and the size of each page is the bst key. */
+	struct btree vo_pages;
+	/* total length of the vm_object in bytes. */
+	size_t vo_size;
+};
+
+extern kern_status_t vm_object_type_init(void);
+extern struct vm_object *vm_object_cast(struct object *obj);
+
+/* create a vm_object with the specified length in bytes and protection flags.
+ * the length will be automatically rounded up to the nearest vm_object page
+ * order size. the actual page frames themselves won't be allocated until
+ * they are mapped and accessed. */
+extern struct vm_object *vm_object_create(
+	const char *name,
+	size_t name_len,
+	size_t data_len,
+	vm_prot_t prot);
+
+/* create a vm_object that represents the specified range of physical memory.
+ * the length will be automatically rounded up to the nearest vm_object page
+ * order size.
+ * NOTE this function assumes that the physical memory has already been
+ * reserved, and is not in use by any other kernel component. */
+extern struct vm_object *vm_object_create_in_place(
+	const char *name,
+	size_t name_len,
+	phys_addr_t base,
+	size_t data_len,
+	vm_prot_t prot);
+
+extern struct vm_page *vm_object_get_page(
+	const struct vm_object *vo,
+	off_t offset);
+
+extern struct vm_page *vm_object_alloc_page(
+	struct vm_object *vo,
+	off_t offset,
+	enum vm_page_order size);
+
+extern kern_status_t vm_object_read(
+	struct vm_object *vo,
+	void *dst,
+	off_t offset,
+	size_t count,
+	size_t *nr_read);
+extern kern_status_t vm_object_write(
+	struct vm_object *vo,
+	const void *dst,
+	off_t offset,
+	size_t count,
+	size_t *nr_written);
+extern kern_status_t vm_object_copy(
+	struct vm_object *dst,
+	off_t dst_offset,
+	struct vm_object *src,
+	off_t src_offset,
+	size_t count,
+	size_t *nr_copied);
+
+DEFINE_OBJECT_LOCK_FUNCTION(vm_object, vo_base)
+
+#endif

include/kernel/vm-region.h

@@ -0,0 +1,191 @@
+#ifndef KERNEL_VM_REGION_H_
+#define KERNEL_VM_REGION_H_
+
+#include <kernel/object.h>
+#include <kernel/pmap.h>
+#include <kernel/vm.h>
+
+#define VM_REGION_NAME_MAX 64
+#define VM_REGION_COPY_ALL ((size_t)-1)
+
+struct vm_region;
+struct vm_object;
+
+enum vm_region_status {
+	VM_REGION_DEAD = 0,
+	VM_REGION_ONLINE,
+};
+
+enum vm_region_entry_type {
+	VM_REGION_ENTRY_NONE = 0,
+	VM_REGION_ENTRY_REGION,
+	VM_REGION_ENTRY_MAPPING,
+};
+
+struct vm_region_entry {
+	union {
+		struct btree_node e_node;
+		/* this entry is only used to queue vm-region objects for
+		 * recursive cleanup */
+		struct queue_entry e_entry;
+	};
+	struct vm_region_entry *e_parent;
+	enum vm_region_entry_type e_type;
+	/* absolute address of this entry */
+	virt_addr_t e_address;
+	/* offset in bytes of this entry within its immediate parent. */
+	off_t e_offset;
+	/* size of the entry in bytes */
+	size_t e_size;
+};
+
+struct vm_region_mapping {
+	struct vm_region_entry m_entry;
+	struct vm_object *m_object;
+
+	/* used to link to vm_object->vo_mappings */
+	struct queue_entry m_object_entry;
+
+	vm_prot_t m_prot;
+	/* offset in bytes to the start of the object data that was mapped */
+	off_t m_object_offset;
+};
+
+struct vm_region {
+	struct object vr_base;
+	enum vm_region_status vr_status;
+	struct vm_region_entry vr_entry;
+
+	char vr_name[VM_REGION_NAME_MAX];
+
+	/* btree of struct vm_region_entry.
+	 * sibling entries cannot overlap each other, and child entries must
+	 * be entirely contained within their immediate parent entry. */
+	struct btree vr_entries;
+
+	/* memory protection restriction mask.
+	 * any mapping in this region, or any of its children, cannot use
+	 * protection flags that are not set in this mask.
+	 * for example, if VM_PROT_EXEC is /not/ set here, no mapping
+	 * can be created in this region or any child region with VM_PROT_EXEC
+	 * set. */
+	vm_prot_t vr_prot;
+
+	/* the physical address space in which mappings in this region (and
+	 * its children) are created */
+	pmap_t vr_pmap;
+};
+
+extern kern_status_t vm_region_type_init(void);
+extern struct vm_region *vm_region_cast(struct object *obj);
+
+/* create a new vm-region, optionally within a parent region.
+ * `offset` is the byte offset within the parent region where the new region
+ * should start.
+ * if no parent is specified, `offset` is the absolute virtual address of the
+ * start of the region.
+ * in both cases, `len` is the length of the new region in bytes. */
+extern kern_status_t vm_region_create(
+	struct vm_region *parent,
+	const char *name,
+	size_t name_len,
+	off_t offset,
+	size_t region_len,
+	vm_prot_t prot,
+	struct vm_region **out);
+
+/* recursively kills a given region and all of its sub-regions.
+ * when a region is killed, all of its mappings are unmapped, and any further
+ * operations on the region are denied. however, all handles and references to
+ * the region (any any sub-region) remain valid, and no kernel memory is
+ * de-allocated.
+ * the memory used by the vm-region object itself is de-allocated when the last
+ * handle/reference to the object is released.
+ * this function should be called with `region` locked.
+ */
+extern kern_status_t vm_region_kill(
+	struct vm_region *region,
+	unsigned long *lock_flags);
+
+/* map a vm-object into a vm-region.
+ * [region_offset,length] must fall within exactly one region, and cannot span
+ * multiple sibling regions.
+ * if [region_offset,length] falls within a child region, the map operation
+ * will be transparently redirected to the relevant region.
+ * `prot` must be allowed both by the region into which the mapping is being
+ * created AND the vm-object being mapped. */
+extern kern_status_t vm_region_map_object(
+	struct vm_region *region,
+	off_t region_offset,
+	struct vm_object *object,
+	off_t object_offset,
+	size_t length,
+	vm_prot_t prot,
+	virt_addr_t *out);
+
+extern kern_status_t vm_region_unmap(
+	struct vm_region *region,
+	off_t region_offset,
+	size_t length);
+
+extern bool vm_region_validate_access(
+	struct vm_region *region,
+	off_t offset,
+	size_t len,
+	vm_prot_t prot);
+
+/* find the mapping corresponding to the given virtual address, and page-in the
+ * necessary vm_page to allow the memory access to succeed. if the relevant
+ * vm-object page hasn't been allocated yet, it will be allocated here. */
+extern kern_status_t vm_region_demand_map(
+	struct vm_region *region,
+	virt_addr_t addr,
+	enum pmap_fault_flags flags);
+
+/* get the absolute base virtual address of a region within its
+ * parent/ancestors. */
+extern virt_addr_t vm_region_get_base_address(const struct vm_region *region);
+
+extern void vm_region_dump(struct vm_region *region);
+
+/* read data from the user-space area of a vm-region into a kernel-mode buffer
+ */
+extern kern_status_t vm_region_read_kernel(
+	struct vm_region *src_region,
+	virt_addr_t src_ptr,
+	size_t count,
+	void *dest,
+	size_t *nr_read);
+
+/* write data to the user-space area of a vm-region from a kernel-mode buffer
+ */
+extern kern_status_t vm_region_write_kernel(
+	struct vm_region *dst_region,
+	virt_addr_t dst_ptr,
+	size_t count,
+	const void *src,
+	size_t *nr_written);
+
+extern kern_status_t vm_region_memmove(
+	struct vm_region *dest_region,
+	virt_addr_t dest_ptr,
+	struct vm_region *src_region,
+	virt_addr_t src_ptr,
+	size_t count,
+	size_t *nr_moved);
+
+extern kern_status_t vm_region_memmove_v(
+	struct vm_region *dest_region,
+	size_t dest_offset,
+	const kern_iovec_t *dest,
+	size_t nr_dest,
+	struct vm_region *src_region,
+	size_t src_offset,
+	const kern_iovec_t *src,
+	size_t nr_src,
+	size_t bytes_to_move,
+	size_t *nr_bytes_moved);
+
+DEFINE_OBJECT_LOCK_FUNCTION(vm_region, vr_base)
+
+#endif

include/kernel/vm.h

@@ -1,14 +1,14 @@
-#ifndef MANGO_VM_H_
+#ifndef KERNEL_VM_H_
-#define MANGO_VM_H_
+#define KERNEL_VM_H_
 
-#include <stddef.h>
+#include <kernel/bitmap.h>
-#include <mango/types.h>
+#include <kernel/btree.h>
+#include <kernel/locks.h>
+#include <kernel/machine/vm.h>
+#include <kernel/queue.h>
+#include <kernel/types.h>
 #include <mango/status.h>
-#include <mango/queue.h>
+#include <stddef.h>
-#include <mango/btree.h>
-#include <mango/bitmap.h>
-#include <mango/locks.h>
-#include <mango/machine/vm.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -17,13 +17,13 @@ extern "C" {
 struct bcache;
 
 /* maximum number of NUMA nodes */
-#define VM_MAX_NODES 64
+#define VM_MAX_NODES            64
 /* maximum number of memory zones per node */
-#define VM_MAX_ZONES (VM_ZONE_MAX + 1)
+#define VM_MAX_ZONES            (VM_ZONE_MAX + 1)
 /* maximum number of supported page orders */
-#define VM_MAX_PAGE_ORDERS (VM_PAGE_MAX_ORDER + 1)
+#define VM_MAX_PAGE_ORDERS      (VM_PAGE_MAX_ORDER + 1)
 /* maximum number of sparse memory sectors */
-#define VM_MAX_SECTORS 8192
+#define VM_MAX_SECTORS          8192
 
 /* maximum number of disk sectors that can be stored in a single
    page. AKA the number of bits in the sector bitmap.
@@ -33,44 +33,32 @@ struct bcache;
 #define VM_CHECK_ALIGN(p, mask) ((((p) & (mask)) == (p)) ? 1 : 0)
 
 #define VM_CACHE_INITIALISED(c) ((c)->c_obj_count != 0)
-#define VM_PAGE_IS_FREE(pg) (((pg)->p_flags & (VM_PAGE_RESERVED | VM_PAGE_ALLOC)) == 0)
+#define VM_PAGE_IS_FREE(pg)                                                    \
+	(((pg)->p_flags & (VM_PAGE_RESERVED | VM_PAGE_ALLOC)) == 0)
 
-#define vm_page_foreach(pg, i) \
+#define vm_page_foreach(pg, i)                                                 \
 	for (struct vm_page *i = (pg); i; i = vm_page_get_next_tail(i))
 
 typedef phys_addr_t vm_alignment_t;
 typedef unsigned int vm_node_id_t;
 
-struct vm_object {
-	unsigned int reserved;
-};
-
 enum vm_model {
 	VM_MODEL_FLAT = 1,
 	VM_MODEL_SPARSE,
 };
 
-enum vm_prot {
-	VM_PROT_READ    = 0x01u,
-	VM_PROT_WRITE   = 0x02u,
-	VM_PROT_EXEC    = 0x04u,
-	VM_PROT_USER    = 0x08u,
-	VM_PROT_SVR     = 0x10u,
-	VM_PROT_NOCACHE = 0x20u,
-};
-
 enum vm_flags {
-	VM_NORMAL  = 0x00u,
+	VM_NORMAL = 0x00u,
 	VM_GET_DMA = 0x01u,
 };
 
 enum vm_zone_id {
-	/* NOTE that these are used as indices into the node_zones array in vm/zone.c
+	/* NOTE that these are used as indices into the node_zones array in
-	   they need to be continuous, and must start at 0!
+	   vm/zone.c they need to be continuous, and must start at 0!
 
 	   not all of these zones are implemented for every architecture. */
-	VM_ZONE_DMA     = 0u,
+	VM_ZONE_DMA = 0u,
-	VM_ZONE_NORMAL  = 1u,
+	VM_ZONE_NORMAL = 1u,
 	VM_ZONE_HIGHMEM = 2u,
 };
 
@@ -108,27 +96,28 @@ enum vm_page_order {
 };
 
 enum vm_page_flags {
-	/* page is reserved (probably by a call to memblock_reserve()) and cannot be
+	/* page is reserved (probably by a call to memblock_reserve()) and
-	   returned by any allocation function */
+	   cannot be returned by any allocation function */
-	VM_PAGE_RESERVED   = 0x01u,
+	VM_PAGE_RESERVED = 0x01u,
 	/* page has been allocated by a zone's buddy allocator, and is in-use */
-	VM_PAGE_ALLOC      = 0x02u,
+	VM_PAGE_ALLOC = 0x02u,
 	/* page is the first page of a huge-page */
-	VM_PAGE_HEAD       = 0x04u,
+	VM_PAGE_HEAD = 0x04u,
 	/* page is part of a huge-page */
-	VM_PAGE_HUGE       = 0x08u,
+	VM_PAGE_HUGE = 0x08u,
-	/* page is holding cached data from secondary storage, and can be freed if necessary (and not dirty). */
+	/* page is holding cached data from secondary storage, and can be freed
-	VM_PAGE_CACHE      = 0x10u,
+	 * if necessary (and not dirty). */
+	VM_PAGE_CACHE = 0x10u,
 };
 
 enum vm_memory_region_status {
-	VM_REGION_FREE		= 0x01u,
+	VM_REGION_FREE = 0x01u,
-	VM_REGION_RESERVED	= 0x02u,
+	VM_REGION_RESERVED = 0x02u,
 };
 
 enum vm_cache_flags {
 	VM_CACHE_OFFSLAB = 0x01u,
-	VM_CACHE_DMA     = 0x02u
+	VM_CACHE_DMA = 0x02u
 };
 
 struct vm_zone_descriptor {
@@ -151,12 +140,6 @@ struct vm_pg_data {
 	struct vm_zone pg_zones[VM_MAX_ZONES];
 };
 
-struct vm_region {
-	enum vm_memory_region_status r_status;
-	phys_addr_t r_base;
-	phys_addr_t r_limit;
-};
-
 struct vm_cache {
 	const char *c_name;
 	enum vm_cache_flags c_flags;
@@ -204,7 +187,7 @@ struct vm_slab {
 	     - s_freelist[s_free] should be set to the previous value of s_free.
 	   this is commented as it as flexible arrays are not supported in c++.
 	   */
-	//unsigned int s_freelist[];
+	// unsigned int s_freelist[];
 };
 
 struct vm_page {
@@ -231,20 +214,25 @@ struct vm_page {
 	/* multi-purpose list/tree entry.
 	   the owner of the page can decide what to do with this.
 	   some examples:
-	     - the buddy allocator uses this to maintain its per-zone free-page lists.
+	     - the buddy allocator uses this to maintain its per-zone free-page
-	     - the block cache uses this to maintain a tree of pages keyed by block number.
+	       lists.
-        */
+	     - vm_object uses it to maintain a btree of allocated pages keyed
+	       by offset/size.
+	     - the block cache uses this to maintain a tree of pages keyed by
+	       block number.
+	*/
 	union {
 		struct queue_entry p_list;
 		struct btree_node p_bnode;
 
-		/* btree_node contains three pointers, so provide three pointer-sized integers for
+		/* btree_node contains three pointers, so provide three
-		   use if p_bnode isn't needed. */
+		   pointer-sized integers for use if p_bnode isn't needed. */
 		uintptr_t priv1[3];
 	};
 
 	union {
-		/* used by bcache when sector size is < page size. bitmap of present/missing sectors */
+		/* used by bcache when sector size is < page size. bitmap of
+		 * present/missing sectors */
 		DECLARE_BITMAP(p_blockbits, VM_MAX_SECTORS_PER_PAGE);
 		uint32_t p_priv2;
 	};
@@ -252,10 +240,12 @@ struct vm_page {
 	union {
 		/* sector address, used by bcache */
 		sectors_t p_blockid;
+		/* offset of this page within the vm_object it is a part of */
+		off_t p_vmo_offset;
 
 		uint32_t p_priv3[2];
 	};
-} __attribute__((aligned(2 * sizeof(unsigned long))));
+} __aligned(2 * sizeof(unsigned long));
 
 /* represents a sector of memory, containing its own array of vm_pages.
    this struct is used under the sparse memory model, instead of the
@@ -272,39 +262,58 @@ struct vm_sector {
 	struct vm_page *s_pages;
 };
 
-extern kern_status_t vm_bootstrap(const struct vm_zone_descriptor *zones, size_t nr_zones);
+extern kern_status_t vm_bootstrap(
+	const struct vm_zone_descriptor *zones,
+	size_t nr_zones);
 extern enum vm_model vm_memory_model(void);
 extern void vm_set_memory_model(enum vm_model model);
 
 extern struct vm_pg_data *vm_pg_data_get(vm_node_id_t node);
 
-extern phys_addr_t vm_virt_to_phys(void *p);
+extern phys_addr_t vm_virt_to_phys(const void *p);
 extern void *vm_phys_to_virt(phys_addr_t p);
 
-extern void vm_page_init_array();
+extern size_t vm_page_order_to_bytes(enum vm_page_order order);
+extern size_t vm_page_order_to_pages(enum vm_page_order order);
+extern vm_alignment_t vm_page_order_to_alignment(enum vm_page_order order);
+extern void vm_page_init_array(void);
 extern struct vm_page *vm_page_get(phys_addr_t addr);
 extern phys_addr_t vm_page_get_paddr(struct vm_page *pg);
 extern struct vm_zone *vm_page_get_zone(struct vm_page *pg);
 extern void *vm_page_get_vaddr(struct vm_page *pg);
 extern size_t vm_page_get_pfn(struct vm_page *pg);
-extern size_t vm_page_order_to_bytes(enum vm_page_order order);
+static inline size_t vm_page_get_size_bytes(const struct vm_page *pg)
-extern size_t vm_page_order_to_pages(enum vm_page_order order);
+{
-extern vm_alignment_t vm_page_order_to_alignment(enum vm_page_order order);
+	return vm_page_order_to_bytes(pg->p_order);
-extern struct vm_page *vm_page_alloc(enum vm_page_order order, enum vm_flags flags);
+}
+extern struct vm_page *vm_page_alloc(
+	enum vm_page_order order,
+	enum vm_flags flags);
 extern void vm_page_free(struct vm_page *pg);
 
-extern int vm_page_split(struct vm_page *pg, struct vm_page **a, struct vm_page **b);
+extern int vm_page_split(
+	struct vm_page *pg,
+	struct vm_page **a,
+	struct vm_page **b);
 extern struct vm_page *vm_page_merge(struct vm_page *a, struct vm_page *b);
 extern struct vm_page *vm_page_get_buddy(struct vm_page *pg);
 extern struct vm_page *vm_page_get_next_tail(struct vm_page *pg);
 
 extern size_t vm_bytes_to_pages(size_t bytes);
 
-extern void vm_zone_init(struct vm_zone *z, const struct vm_zone_descriptor *zone_info);
+extern void vm_zone_init(
-extern struct vm_page *vm_zone_alloc_page(struct vm_zone *z, enum vm_page_order order, enum vm_flags flags);
+	struct vm_zone *z,
+	const struct vm_zone_descriptor *zone_info);
+extern struct vm_page *vm_zone_alloc_page(
+	struct vm_zone *z,
+	enum vm_page_order order,
+	enum vm_flags flags);
 extern void vm_zone_free_page(struct vm_zone *z, struct vm_page *pg);
 
-extern struct vm_cache *vm_cache_create(const char *name, size_t objsz, enum vm_cache_flags flags);
+extern struct vm_cache *vm_cache_create(
+	const char *name,
+	size_t objsz,
+	enum vm_cache_flags flags);
 extern void vm_cache_init(struct vm_cache *cache);
 extern void vm_cache_destroy(struct vm_cache *cache);
 extern void *vm_cache_alloc(struct vm_cache *cache, enum vm_flags flags);
@@ -330,15 +339,18 @@ extern size_t vm_page_get_pfn_sparse(struct vm_page *pg);
 #endif
 
 #ifdef __cplusplus
-inline void *operator new(size_t count, void *p) { return p; }
+inline void *operator new(size_t count, void *p)
+{
+	return p;
+}
 
-#define kmalloc_object(objtype, flags, ...)        \
+#define kmalloc_object(objtype, flags, ...)                                    \
-        __extension__({                            \
+	__extension__({                                                        \
-	void *p = kmalloc(sizeof(objtype), flags); \
+		void *p = kmalloc(sizeof(objtype), flags);                     \
-	if (p) {                                   \
+		if (p) {                                                       \
-		new (p) objtype(__VA_ARGS__);      \
+			new (p) objtype(__VA_ARGS__);                          \
-	}                                          \
+		}                                                              \
-	(objtype *)p;                              \
+		(objtype *)p;                                                  \
 	})
 
 #endif

include/mango/locks.h

@@ -1,25 +0,0 @@
-#ifndef MANGO_LOCKS_H_
-#define MANGO_LOCKS_H_
-
-#include <mango/compiler.h>
-#include <mango/machine/hwlock.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef __aligned(8) ml_hwlock_t spin_lock_t;
-
-#define SPIN_LOCK_INIT ML_HWLOCK_INIT
-
-#define spin_lock(lck) ml_hwlock_lock(lck);
-#define spin_unlock(lck) ml_hwlock_unlock(lck);
-
-#define spin_lock_irqsave(lck, flags) ml_hwlock_lock_irqsave(lck, flags);
-#define spin_unlock_irqrestore(lck, flags) ml_hwlock_unlock_irqrestore(lck, flags);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

include/mango/object.h

@@ -1,71 +0,0 @@
-#ifndef MANGO_OBJECT_H_
-#define MANGO_OBJECT_H_
-
-#include <mango/flags.h>
-#include <mango/locks.h>
-#include <mango/status.h>
-#include <mango/vm.h>
-#include <stddef.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define OBJECT_MAGIC    0xBADDCAFE
-#define OBJECT_NAME_MAX 64
-#define OBJECT_PATH_MAX 256
-
-#define OBJECT_CAST(to_type, to_type_member, p)                                \
-	((to_type *)((uintptr_t)p) - offsetof(to_type, to_type_member))
-#define OBJECT_C_CAST(c_type, c_type_member, obj_type, objp)                   \
-	OBJECT_IS_TYPE(objp, obj_type)                                         \
-	? OBJECT_CAST(c_type, c_type_member, (objp)) : NULL
-#define OBJECT_IS_TYPE(obj, type_ptr) ((obj)->ob_type == (type_ptr))
-
-struct object;
-struct object_attrib;
-
-enum object_type_flags {
-	OBJTYPE_INIT = 0x01u,
-};
-
-struct object_ops {
-	kern_status_t (*destroy)(struct object *obj);
-};
-
-struct object_type {
-	enum object_type_flags ob_flags;
-	char ob_name[32];
-	unsigned int ob_size;
-	unsigned int ob_header_offset;
-	struct vm_cache ob_cache;
-	struct queue_entry ob_list;
-	struct object_ops ob_ops;
-};
-
-struct object {
-	uint32_t ob_magic;
-	struct object_type *ob_type;
-	spin_lock_t ob_lock;
-	unsigned int ob_refcount;
-	unsigned int ob_handles;
-	struct queue_entry ob_list;
-} __aligned(sizeof(long));
-
-extern kern_status_t object_bootstrap(void);
-extern kern_status_t object_type_register(struct object_type *p);
-extern kern_status_t object_type_unregister(struct object_type *p);
-
-extern struct object *object_create(struct object_type *type);
-extern struct object *object_ref(struct object *obj);
-extern void object_deref(struct object *obj);
-extern void object_lock(struct object *obj);
-extern void object_unlock(struct object *obj);
-extern void object_lock_irqsave(struct object *obj, unsigned long *flags);
-extern void object_unlock_irqrestore(struct object *obj, unsigned long flags);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

include/mango/panic.h

@@ -1,12 +0,0 @@
-#ifndef MANGO_PANIC_H_
-#define MANGO_PANIC_H_
-
-#include <mango/compiler.h>
-
-struct cpu_context;
-
-#define panic(...) panic_irq(NULL, __VA_ARGS__)
-
-extern void __noreturn panic_irq(struct cpu_context *ctx, const char *fmt, ...);
-
-#endif

include/mango/pmap.h

@@ -1,42 +0,0 @@
-#ifndef MANGO_PMAP_H_
-#define MANGO_PMAP_H_
-
-/* all the functions declared in this file are defined in arch/xyz/pmap.c */
-
-#include <mango/vm.h>
-#include <mango/status.h>
-#include <mango/machine/pmap.h>
-#include <stddef.h>
-
-#define PFN(x) ((x) >> VM_PAGE_SHIFT)
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef ml_pmap_t pmap_t;
-typedef ml_pfn_t pfn_t;
-
-enum pmap_flags {
-	PMAP_NORMAL = 0x00u,
-	PMAP_HUGEPAGE = 0x01u,
-};
-
-extern void pmap_bootstrap(void);
-extern pmap_t get_kernel_pmap(void);
-
-extern pmap_t pmap_create(void);
-extern void pmap_destroy(pmap_t pmap);
-extern void pmap_switch(pmap_t pmap);
-
-extern kern_status_t pmap_add(pmap_t pmap, void *p, pfn_t pfn, enum vm_prot prot, enum pmap_flags flags);
-extern kern_status_t pmap_add_block(pmap_t pmap, void *p, pfn_t pfn, size_t len, enum vm_prot prot, enum pmap_flags flags);
-
-extern kern_status_t pmap_remove(pmap_t pmap, void *p);
-extern kern_status_t pmap_remove_range(pmap_t pmap, void *p, size_t len);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

include/mango/printk.h

@@ -1,17 +0,0 @@
-#ifndef MANGO_PRINTK_H_
-#define MANGO_PRINTK_H_
-
-#include <mango/console.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-extern void early_printk_init(struct console *con);
-extern int printk(const char *format, ...);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

include/mango/types.h

@@ -1,14 +0,0 @@
-#ifndef MANGO_TYPES_H_
-#define MANGO_TYPES_H_
-
-#include <stdint.h>
-
-#define CYCLES_MAX UINT64_MAX
-
-typedef uintptr_t phys_addr_t;
-typedef uint64_t cycles_t;
-typedef uint64_t sectors_t;
-
-typedef unsigned int umode_t;
-
-#endif

init/init.c

@@ -1,5 +1,4 @@
-#include <mango/init.h>
+#include <kernel/init.h>
-
 
 int do_initcalls(void)
 {

init/main.c

@@ -1,15 +1,20 @@
-#include <mango/arg.h>
+#include <kernel/arg.h>
-#include <mango/clock.h>
+#include <kernel/bsp.h>
-#include <mango/cpu.h>
+#include <kernel/channel.h>
-#include <mango/init.h>
+#include <kernel/clock.h>
-#include <mango/input.h>
+#include <kernel/cpu.h>
-#include <mango/libc/stdio.h>
+#include <kernel/handle.h>
-#include <mango/machine/init.h>
+#include <kernel/init.h>
-#include <mango/object.h>
+#include <kernel/input.h>
-#include <mango/panic.h>
+#include <kernel/libc/stdio.h>
-#include <mango/printk.h>
+#include <kernel/machine/init.h>
-#include <mango/sched.h>
+#include <kernel/object.h>
-#include <mango/test.h>
+#include <kernel/panic.h>
+#include <kernel/port.h>
+#include <kernel/printk.h>
+#include <kernel/sched.h>
+#include <kernel/test.h>
+#include <kernel/vm-object.h>
 #include <stdint.h>
 
 extern unsigned long get_rflags(void);
@@ -23,14 +28,16 @@ void print_kernel_banner(void)
 
 static void hang(void)
 {
-	struct task *self = current_task();
+	// struct task *self = current_task();
-	struct thread *thread = current_thread();
+	// struct thread *thread = current_thread();
 
 	while (1) {
-		printk("[cpu %u, task %u, thread %u]: tick",
+#if 0
+		printk("[cpu %u, task %ld, thread %u]: tick",
 		       this_cpu(),
 		       self->t_id,
 		       thread->tr_id);
+#endif
 		milli_sleep(2000);
 	}
 }
@@ -67,9 +74,43 @@ void kernel_init(uintptr_t arg)
 {
 	ml_init(arg);
 
-	printk("kernel_init() running on processor %u", this_cpu());
+	port_type_init();
+	channel_type_init();
 
-	create_kernel_thread(background_thread);
+	struct boot_module bsp_image = {0};
+	bsp_get_location(&bsp_image);
+
+	tracek("kernel_init() running on processor %u", this_cpu());
+
+	if (!bsp_image.mod_base) {
+		printk("FATAL: no bsp image specified");
+		hang();
+	}
+
+	tracek("bsp image at [0x%llx-0x%llx]",
+	       bsp_image.mod_base,
+	       bsp_image.mod_base + bsp_image.mod_size);
+
+	struct bsp bsp;
+	kern_status_t status = bsp_load(&bsp, &bsp_image);
+	if (status != KERN_OK) {
+		printk("FATAL: bsp image is corrupt/invalid");
+		hang();
+	}
+
+	tracek("bsp image loaded. text=[%06llx-%06llx], data=[%06llx-%06llx], "
+	       "entry=%06llx, vmo=%p",
+	       bsp.bsp_trailer.bsp_text_vaddr,
+	       bsp.bsp_trailer.bsp_text_size + bsp.bsp_trailer.bsp_text_vaddr,
+	       bsp.bsp_trailer.bsp_data_vaddr,
+	       bsp.bsp_trailer.bsp_data_size + bsp.bsp_trailer.bsp_data_vaddr,
+	       bsp.bsp_trailer.bsp_exec_entry,
+	       bsp.bsp_vmo);
+
+	struct task *bootstrap_task = task_create("bootstrap", 9);
+	tracek("created bootstrap task (pid=%u)", bootstrap_task->t_id);
+
+	bsp_launch_async(&bsp, bootstrap_task);
 
 	hang();
 }

kernel/arg.c

@@ -1,6 +1,7 @@
-#include <mango/arg.h>
+#include <kernel/arg.h>
-#include <mango/libc/string.h>
+#include <kernel/libc/ctype.h>
-#include <mango/libc/ctype.h>
+#include <kernel/libc/string.h>
+#include <mango/status.h>
 
 static char g_cmdline[CMDLINE_MAX + 1] = {0};
 
@@ -81,7 +82,6 @@ static char *advance_to_next_arg(char *s, char *max)
 	return s;
 }
 
-
 const char *arg_value(const char *arg_name)
 {
 	char *s = g_cmdline;

kernel/bsp.c

@@ -0,0 +1,320 @@
+#include <kernel/bsp.h>
+#include <kernel/handle.h>
+#include <kernel/printk.h>
+#include <kernel/sched.h>
+#include <kernel/util.h>
+#include <kernel/vm-object.h>
+#include <kernel/vm-region.h>
+
+#define BOOTSTRAP_STACK_SIZE 0x10000
+
+static struct boot_module bsp_location = {0};
+
+void bsp_set_location(const struct boot_module *mod)
+{
+	memcpy(&bsp_location, mod, sizeof bsp_location);
+}
+
+void bsp_get_location(struct boot_module *out)
+{
+	memcpy(out, &bsp_location, sizeof bsp_location);
+}
+
+kern_status_t bsp_load(struct bsp *bsp, const struct boot_module *mod)
+{
+	size_t trailer_offset = mod->mod_size - sizeof(struct bsp_trailer);
+	const char *p = vm_phys_to_virt(mod->mod_base);
+	const struct bsp_trailer *trailer_bigendian
+		= (const struct bsp_trailer *)(p + trailer_offset);
+
+	bsp->bsp_trailer.bsp_magic
+		= big_to_host_u32(trailer_bigendian->bsp_magic);
+	bsp->bsp_trailer.bsp_fs_offset
+		= big_to_host_u64(trailer_bigendian->bsp_fs_offset);
+	bsp->bsp_trailer.bsp_fs_len
+		= big_to_host_u32(trailer_bigendian->bsp_fs_len);
+	bsp->bsp_trailer.bsp_exec_offset
+		= big_to_host_u64(trailer_bigendian->bsp_exec_offset);
+	bsp->bsp_trailer.bsp_exec_len
+		= big_to_host_u32(trailer_bigendian->bsp_exec_len);
+	bsp->bsp_trailer.bsp_text_faddr
+		= big_to_host_u64(trailer_bigendian->bsp_text_faddr);
+	bsp->bsp_trailer.bsp_text_vaddr
+		= big_to_host_u64(trailer_bigendian->bsp_text_vaddr);
+	bsp->bsp_trailer.bsp_text_size
+		= big_to_host_u64(trailer_bigendian->bsp_text_size);
+	bsp->bsp_trailer.bsp_data_faddr
+		= big_to_host_u64(trailer_bigendian->bsp_data_faddr);
+	bsp->bsp_trailer.bsp_data_vaddr
+		= big_to_host_u64(trailer_bigendian->bsp_data_vaddr);
+	bsp->bsp_trailer.bsp_data_size
+		= big_to_host_u64(trailer_bigendian->bsp_data_size);
+	bsp->bsp_trailer.bsp_exec_entry
+		= big_to_host_u64(trailer_bigendian->bsp_exec_entry);
+
+	if (bsp->bsp_trailer.bsp_magic != BSP_MAGIC) {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	bsp->bsp_vmo = vm_object_create_in_place(
+		"bsp",
+		3,
+		mod->mod_base,
+		mod->mod_size,
+		VM_PROT_READ | VM_PROT_EXEC | VM_PROT_USER);
+	if (!bsp->bsp_vmo) {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	return KERN_OK;
+}
+
+static kern_status_t map_executable_dyn(
+	struct bsp *bsp,
+	struct task *task,
+	virt_addr_t *entry)
+{
+	kern_status_t status = KERN_OK;
+	size_t exec_size = 0;
+	if (bsp->bsp_trailer.bsp_text_vaddr > bsp->bsp_trailer.bsp_data_vaddr) {
+		exec_size = bsp->bsp_trailer.bsp_text_vaddr
+		          + bsp->bsp_trailer.bsp_text_size;
+	} else {
+		exec_size = bsp->bsp_trailer.bsp_data_vaddr
+		          + bsp->bsp_trailer.bsp_data_size;
+	}
+
+	struct vm_region *region;
+	status = vm_region_create(
+		task->t_address_space,
+		"exec",
+		4,
+		VM_REGION_ANY_OFFSET,
+		exec_size,
+		VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXEC | VM_PROT_USER,
+		&region);
+	if (status != KERN_OK) {
+		return status;
+	}
+
+	struct vm_object *data = vm_object_create(
+		".data",
+		5,
+		bsp->bsp_trailer.bsp_data_size,
+		VM_PROT_READ | VM_PROT_WRITE | VM_PROT_USER);
+	if (!data) {
+		return KERN_NO_MEMORY;
+	}
+
+	virt_addr_t text_base = 0, data_base = 0;
+
+	off_t text_foffset = bsp->bsp_trailer.bsp_exec_offset
+	                   + bsp->bsp_trailer.bsp_text_faddr;
+	off_t data_foffset = 0;
+	off_t text_voffset = bsp->bsp_trailer.bsp_text_vaddr;
+	off_t data_voffset = bsp->bsp_trailer.bsp_data_vaddr;
+
+#if 0
+	size_t tmp = 0;
+	status = vm_object_copy(
+		data,
+		0,
+		bsp->bsp_vmo,
+		bsp->bsp_trailer.bsp_data_faddr,
+		bsp->bsp_trailer.bsp_data_size,
+		&tmp);
+
+	tracek("read %zuB of data from executable", tmp);
+#endif
+
+	tracek("text_foffset=%06llx, data_foffset=%06llx",
+	       text_foffset,
+	       data_foffset);
+	tracek("text_voffset=%08llx, data_voffset=%08llx",
+	       text_voffset,
+	       data_voffset);
+
+	status = vm_region_map_object(
+		region,
+		text_voffset,
+		bsp->bsp_vmo,
+		text_foffset,
+		bsp->bsp_trailer.bsp_text_size,
+		VM_PROT_READ | VM_PROT_EXEC | VM_PROT_USER,
+		&text_base);
+	if (status != KERN_OK) {
+		return status;
+	}
+
+	status = vm_region_map_object(
+		region,
+		data_voffset,
+		data,
+		data_foffset,
+		bsp->bsp_trailer.bsp_data_size,
+		VM_PROT_READ | VM_PROT_WRITE | VM_PROT_USER,
+		&data_base);
+	if (status != KERN_OK) {
+		return status;
+	}
+
+	tracek("text_base=%08llx, data_base=%08llx", text_base, data_base);
+
+	*entry = text_base + bsp->bsp_trailer.bsp_exec_entry;
+	return KERN_OK;
+}
+
+static kern_status_t map_executable_exec(
+	struct bsp *bsp,
+	struct task *task,
+	virt_addr_t *entry)
+{
+	kern_status_t status = KERN_OK;
+	struct vm_object *data = vm_object_create(
+		".data",
+		5,
+		bsp->bsp_trailer.bsp_data_size,
+		VM_PROT_READ | VM_PROT_WRITE | VM_PROT_USER);
+	if (!data) {
+		return KERN_NO_MEMORY;
+	}
+
+	virt_addr_t text_base = 0, data_base = 0;
+
+	off_t text_foffset = bsp->bsp_trailer.bsp_exec_offset
+	                   + bsp->bsp_trailer.bsp_text_faddr;
+	off_t data_foffset = 0;
+	off_t text_voffset = bsp->bsp_trailer.bsp_text_vaddr;
+	off_t data_voffset = bsp->bsp_trailer.bsp_data_vaddr;
+
+	text_voffset -= vm_region_get_base_address(task->t_address_space);
+	data_voffset -= vm_region_get_base_address(task->t_address_space);
+
+#if 0
+	size_t tmp = 0;
+	status = vm_object_copy(
+		data,
+		0,
+		bsp->bsp_vmo,
+		bsp->bsp_trailer.bsp_data_faddr,
+		bsp->bsp_trailer.bsp_data_size,
+		&tmp);
+
+	tracek("read %zuB of data from executable", tmp);
+#endif
+
+	tracek("text_foffset=%06llx, data_foffset=%06llx",
+	       text_foffset,
+	       data_foffset);
+	tracek("text_voffset=%08llx, data_voffset=%08llx",
+	       text_voffset,
+	       data_voffset);
+
+	status = vm_region_map_object(
+		task->t_address_space,
+		text_voffset,
+		bsp->bsp_vmo,
+		text_foffset,
+		bsp->bsp_trailer.bsp_text_size,
+		VM_PROT_READ | VM_PROT_EXEC | VM_PROT_USER,
+		&text_base);
+	if (status != KERN_OK) {
+		return status;
+	}
+
+	status = vm_region_map_object(
+		task->t_address_space,
+		data_voffset,
+		data,
+		data_foffset,
+		bsp->bsp_trailer.bsp_data_size,
+		VM_PROT_READ | VM_PROT_WRITE | VM_PROT_USER,
+		&data_base);
+	if (status != KERN_OK) {
+		return status;
+	}
+
+	tracek("text_base=%08llx, data_base=%08llx", text_base, data_base);
+
+	*entry = bsp->bsp_trailer.bsp_exec_entry;
+	return KERN_OK;
+}
+
+kern_status_t bsp_launch_async(struct bsp *bsp, struct task *task)
+{
+	virt_addr_t stack_buffer, bsp_data_base;
+	virt_addr_t entry, sp;
+	kern_status_t status;
+
+	struct vm_object *user_stack = vm_object_create(
+		"stack",
+		5,
+		BOOTSTRAP_STACK_SIZE,
+		VM_PROT_READ | VM_PROT_WRITE | VM_PROT_USER);
+	if (!user_stack) {
+		return KERN_NO_ENTRY;
+	}
+
+	status = vm_region_map_object(
+		task->t_address_space,
+		VM_REGION_ANY_OFFSET,
+		user_stack,
+		0,
+		BOOTSTRAP_STACK_SIZE,
+		VM_PROT_READ | VM_PROT_WRITE | VM_PROT_USER,
+		&stack_buffer);
+
+	if (status != KERN_OK) {
+		return status;
+	}
+
+	status = vm_region_map_object(
+		task->t_address_space,
+		VM_REGION_ANY_OFFSET,
+		bsp->bsp_vmo,
+		0,
+		bsp->bsp_trailer.bsp_exec_offset,
+		VM_PROT_READ | VM_PROT_USER,
+		&bsp_data_base);
+
+	if (status != KERN_OK) {
+		return status;
+	}
+
+	status = map_executable_exec(bsp, task, &entry);
+	if (status != KERN_OK) {
+		return status;
+	}
+#ifdef TRACE
+	vm_region_dump(task->t_address_space);
+#endif
+
+	sp = stack_buffer + BOOTSTRAP_STACK_SIZE;
+	tracek("bootstrap: entry=%llx, sp=%llx", entry, sp);
+
+	kern_handle_t self, self_address_space;
+	task_open_handle(task, &task->t_base, 0, &self);
+	task_open_handle(
+		task,
+		&task->t_address_space->vr_base,
+		0,
+		&self_address_space);
+
+	const uintptr_t args[] = {
+		0,                    // int argc
+		0,                    // const char ** argv
+		self,                 // kern_handle_t task
+		self_address_space,   // kern_handle_t address_space
+
+		/* this parameter is specific to the bsp bootstrap program, so
+	         * that it can access the rest of the bsp image. */
+		bsp_data_base,
+	};
+	const size_t nr_args = sizeof args / sizeof args[0];
+
+	struct thread *init_thread = task_create_thread(task);
+	thread_init_user(init_thread, entry, sp, args, nr_args);
+	schedule_thread_on_cpu(init_thread);
+
+	return KERN_OK;
+}

kernel/channel.c

@@ -0,0 +1,379 @@
+#include <kernel/channel.h>
+#include <kernel/msg.h>
+#include <kernel/port.h>
+#include <kernel/util.h>
+#include <kernel/vm-region.h>
+
+#define CHANNEL_CAST(p) OBJECT_C_CAST(struct channel, c_base, &channel_type, p)
+
+static struct object_type channel_type = {
+	.ob_name = "channel",
+	.ob_size = sizeof(struct channel),
+	.ob_header_offset = offsetof(struct channel, c_base),
+};
+
+BTREE_DEFINE_SIMPLE_GET(struct msg, msgid_t, msg_node, msg_id, get_msg_with_id)
+
+kern_status_t channel_type_init(void)
+{
+	return object_type_register(&channel_type);
+}
+
+struct channel *channel_cast(struct object *obj)
+{
+	return CHANNEL_CAST(obj);
+}
+
+extern struct channel *channel_create(void)
+{
+	struct object *channel_object = object_create(&channel_type);
+	if (!channel_object) {
+		return NULL;
+	}
+
+	struct channel *channel = CHANNEL_CAST(channel_object);
+
+	return channel;
+}
+
+static bool try_enqueue(struct btree *tree, struct msg *msg)
+{
+	if (!tree->b_root) {
+		tree->b_root = &msg->msg_node;
+		btree_insert_fixup(tree, &msg->msg_node);
+		return true;
+	}
+
+	struct btree_node *cur = tree->b_root;
+	while (1) {
+		struct msg *cur_node
+			= BTREE_CONTAINER(struct msg, msg_node, cur);
+		struct btree_node *next = NULL;
+
+		if (msg->msg_id > cur_node->msg_id) {
+			next = btree_right(cur);
+
+			if (!next) {
+				btree_put_right(cur, &msg->msg_node);
+				break;
+			}
+		} else if (msg->msg_id < cur_node->msg_id) {
+			next = btree_left(cur);
+
+			if (!next) {
+				btree_put_left(cur, &msg->msg_node);
+				break;
+			}
+		} else {
+			return false;
+		}
+
+		cur = next;
+	}
+
+	btree_insert_fixup(tree, &msg->msg_node);
+	return true;
+}
+
+static void kmsg_reply_error(
+	struct msg *msg,
+	kern_status_t status,
+	unsigned long *lock_flags)
+{
+	msg->msg_status = KMSG_REPLY_SENT;
+	msg->msg_sender_port->p_status = PORT_READY;
+	msg->msg_result = status;
+	thread_awaken(msg->msg_sender_thread);
+	spin_unlock_irqrestore(&msg->msg_lock, *lock_flags);
+}
+
+static struct msg *get_next_msg(
+	struct channel *channel,
+	unsigned long *lock_flags)
+{
+	struct btree_node *cur = btree_first(&channel->c_msg);
+	while (cur) {
+		struct msg *msg = BTREE_CONTAINER(struct msg, msg_node, cur);
+		spin_lock_irqsave(&msg->msg_lock, lock_flags);
+		if (msg->msg_status == KMSG_WAIT_RECEIVE) {
+			msg->msg_status = KMSG_WAIT_REPLY;
+			msg->msg_sender_port->p_status = PORT_REPLY_BLOCKED;
+			return msg;
+		}
+
+		spin_unlock_irqrestore(&msg->msg_lock, *lock_flags);
+		cur = btree_next(cur);
+	}
+
+	return NULL;
+}
+
+extern kern_status_t channel_enqueue_msg(
+	struct channel *channel,
+	struct msg *msg)
+{
+	fill_random(&msg->msg_id, sizeof msg->msg_id);
+	while (!try_enqueue(&channel->c_msg, msg)) {
+		msg->msg_id++;
+	}
+
+	wakeup_one(&channel->c_wq);
+
+	return KERN_OK;
+}
+
+extern kern_status_t channel_recv_msg(
+	struct channel *channel,
+	kern_msg_t *out_msg,
+	unsigned long *irq_flags)
+{
+	struct wait_item waiter;
+	struct thread *self = current_thread();
+	struct msg *msg = NULL;
+	unsigned long msg_lock_flags;
+
+	wait_item_init(&waiter, self);
+	for (;;) {
+		thread_wait_begin(&waiter, &channel->c_wq);
+		msg = get_next_msg(channel, &msg_lock_flags);
+		if (msg) {
+			break;
+		}
+
+		object_unlock_irqrestore(&channel->c_base, *irq_flags);
+		schedule(SCHED_NORMAL);
+		object_lock_irqsave(&channel->c_base, irq_flags);
+	}
+	thread_wait_end(&waiter, &channel->c_wq);
+
+	/* msg is now set to the next message to process */
+
+	struct task *sender = msg->msg_sender_thread->tr_parent;
+	struct task *receiver = self->tr_parent;
+
+	struct vm_region *src = sender->t_address_space,
+			 *dst = receiver->t_address_space;
+
+	unsigned long f;
+	vm_region_lock_pair_irqsave(src, dst, &f);
+
+	kern_status_t status = vm_region_memmove_v(
+		dst,
+		0,
+		out_msg->msg_data,
+		out_msg->msg_data_count,
+		src,
+		0,
+		msg->msg_req.msg_data,
+		msg->msg_req.msg_data_count,
+		VM_REGION_COPY_ALL,
+		NULL);
+
+	if (status != KERN_OK) {
+		kmsg_reply_error(msg, status, &msg_lock_flags);
+		return status;
+	}
+
+	struct handle_table *src_table = sender->t_handles,
+			    *dst_table = receiver->t_handles;
+
+	spin_lock_pair_irqsave(
+		&sender->t_handles_lock,
+		&receiver->t_handles_lock,
+		&f);
+	status = handle_table_transfer(
+		dst,
+		dst_table,
+		out_msg->msg_handles,
+		out_msg->msg_handles_count,
+		src,
+		src_table,
+		msg->msg_req.msg_handles,
+		msg->msg_req.msg_handles_count);
+	spin_unlock_pair_irqrestore(
+		&sender->t_handles_lock,
+		&receiver->t_handles_lock,
+		f);
+	vm_region_unlock_pair_irqrestore(src, dst, f);
+
+	if (status != KERN_OK) {
+		kmsg_reply_error(msg, status, &msg_lock_flags);
+		return status;
+	}
+
+	out_msg->msg_id = msg->msg_id;
+	out_msg->msg_sender = msg->msg_sender_thread->tr_parent->t_id;
+	out_msg->msg_endpoint = msg->msg_sender_port->p_base.ob_id;
+
+	spin_unlock_irqrestore(&msg->msg_lock, msg_lock_flags);
+
+	return KERN_OK;
+}
+
+extern kern_status_t channel_reply_msg(
+	struct channel *channel,
+	msgid_t id,
+	const kern_msg_t *reply,
+	unsigned long *irq_flags)
+{
+	unsigned long msg_lock_flags;
+	struct msg *msg = get_msg_with_id(&channel->c_msg, id);
+	if (!msg) {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	spin_lock_irqsave(&msg->msg_lock, &msg_lock_flags);
+	if (msg->msg_status != KMSG_WAIT_REPLY) {
+		spin_unlock_irqrestore(&msg->msg_lock, msg_lock_flags);
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	struct thread *self = current_thread();
+	/* the task that is about to receive the response */
+	struct task *receiver = msg->msg_sender_thread->tr_parent;
+	/* the task that is about to send the response */
+	struct task *sender = self->tr_parent;
+
+	struct vm_region *src = sender->t_address_space,
+			 *dst = receiver->t_address_space;
+	unsigned long f;
+	vm_region_lock_pair_irqsave(src, dst, &f);
+
+	kern_status_t status = vm_region_memmove_v(
+		dst,
+		0,
+		msg->msg_resp.msg_data,
+		msg->msg_resp.msg_data_count,
+		src,
+		0,
+		reply->msg_data,
+		reply->msg_data_count,
+		VM_REGION_COPY_ALL,
+		NULL);
+
+	if (status != KERN_OK) {
+		kmsg_reply_error(msg, status, &msg_lock_flags);
+		return status;
+	}
+
+	struct handle_table *src_table = sender->t_handles,
+			    *dst_table = receiver->t_handles;
+
+	spin_lock_pair_irqsave(
+		&sender->t_handles_lock,
+		&receiver->t_handles_lock,
+		&f);
+	status = handle_table_transfer(
+		dst,
+		dst_table,
+		msg->msg_resp.msg_handles,
+		msg->msg_resp.msg_handles_count,
+		src,
+		src_table,
+		reply->msg_handles,
+		reply->msg_handles_count);
+	spin_unlock_pair_irqrestore(
+		&sender->t_handles_lock,
+		&receiver->t_handles_lock,
+		f);
+	vm_region_unlock_pair_irqrestore(src, dst, f);
+
+	if (status != KERN_OK) {
+		kmsg_reply_error(msg, status, &msg_lock_flags);
+		return status;
+	}
+
+	kmsg_reply_error(msg, KERN_OK, &msg_lock_flags);
+
+	return KERN_OK;
+}
+
+extern kern_status_t channel_read_msg(
+	struct channel *channel,
+	msgid_t id,
+	size_t offset,
+	struct vm_region *dest_region,
+	const kern_iovec_t *dest_iov,
+	size_t dest_iov_count,
+	size_t *nr_read)
+{
+	unsigned long msg_lock_flags;
+	struct msg *msg = get_msg_with_id(&channel->c_msg, id);
+	if (!msg) {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	spin_lock_irqsave(&msg->msg_lock, &msg_lock_flags);
+	if (msg->msg_status != KMSG_WAIT_REPLY) {
+		spin_unlock_irqrestore(&msg->msg_lock, msg_lock_flags);
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	struct vm_region *src_region
+		= msg->msg_sender_thread->tr_parent->t_address_space;
+
+	unsigned long f;
+	vm_region_lock_pair_irqsave(src_region, dest_region, &f);
+
+	kern_status_t status = vm_region_memmove_v(
+		dest_region,
+		0,
+		dest_iov,
+		dest_iov_count,
+		src_region,
+		offset,
+		msg->msg_req.msg_data,
+		msg->msg_req.msg_data_count,
+		VM_REGION_COPY_ALL,
+		nr_read);
+	vm_region_unlock_pair_irqrestore(src_region, dest_region, f);
+
+	spin_unlock_irqrestore(&msg->msg_lock, msg_lock_flags);
+
+	return status;
+}
+
+extern kern_status_t channel_write_msg(
+	struct channel *channel,
+	msgid_t id,
+	size_t offset,
+	struct vm_region *src_region,
+	const kern_iovec_t *src_iov,
+	size_t src_iov_count,
+	size_t *nr_written)
+{
+	unsigned long msg_lock_flags;
+	struct msg *msg = get_msg_with_id(&channel->c_msg, id);
+	if (!msg) {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	spin_lock_irqsave(&msg->msg_lock, &msg_lock_flags);
+	if (msg->msg_status != KMSG_WAIT_REPLY) {
+		spin_unlock_irqrestore(&msg->msg_lock, msg_lock_flags);
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	struct vm_region *dest_region
+		= msg->msg_sender_thread->tr_parent->t_address_space;
+
+	unsigned long f;
+	vm_region_lock_pair_irqsave(src_region, dest_region, &f);
+
+	kern_status_t status = vm_region_memmove_v(
+		dest_region,
+		offset,
+		msg->msg_resp.msg_data,
+		msg->msg_resp.msg_data_count,
+		src_region,
+		0,
+		src_iov,
+		src_iov_count,
+		VM_REGION_COPY_ALL,
+		nr_written);
+	vm_region_unlock_pair_irqrestore(src_region, dest_region, f);
+
+	spin_unlock_irqrestore(&msg->msg_lock, msg_lock_flags);
+
+	return status;
+}

kernel/clock.c

@@ -1,6 +1,6 @@
-#include <mango/clock.h>
+#include <kernel/clock.h>
-#include <mango/printk.h>
+#include <kernel/printk.h>
-#include <mango/compiler.h>
+#include <kernel/compiler.h>
 
 static clock_ticks_t ticks_per_sec = 0;
 volatile clock_ticks_t clock_ticks = 0;

kernel/console.c

@@ -1,7 +1,7 @@
-#include <mango/console.h>
+#include <kernel/console.h>
-#include <mango/queue.h>
+#include <kernel/queue.h>
-#include <mango/locks.h>
+#include <kernel/locks.h>
-#include <mango/libc/string.h>
+#include <kernel/libc/string.h>
 
 static struct queue consoles;
 static spin_lock_t consoles_lock = SPIN_LOCK_INIT;

kernel/cpu.c

@@ -1,6 +1,6 @@
-#include <mango/cpu.h>
+#include <kernel/cpu.h>
-#include <mango/percpu.h>
+#include <kernel/percpu.h>
-#include <mango/bitmap.h>
+#include <kernel/bitmap.h>
 
 DECLARE_BITMAP(cpu_available, CPU_MAX);
 DECLARE_BITMAP(cpu_online, CPU_MAX);

kernel/handle.c

@@ -0,0 +1,336 @@
+#include <kernel/handle.h>
+#include <kernel/libc/string.h>
+#include <kernel/object.h>
+#include <kernel/sched.h>
+#include <kernel/util.h>
+#include <kernel/vm-region.h>
+#include <kernel/vm.h>
+#include <mango/types.h>
+
+/* depth=3 gives a maximum of ~66.6 million handles */
+#define MAX_TABLE_DEPTH  3
+#define RESERVED_HANDLES 64
+
+static struct vm_cache handle_table_cache = {
+	.c_name = "handle_table",
+	.c_obj_size = sizeof(struct handle_table),
+};
+
+struct handle_table *handle_table_create(void)
+{
+	if (!VM_CACHE_INITIALISED(&handle_table_cache)) {
+		vm_cache_init(&handle_table_cache);
+	}
+
+	struct handle_table *out
+		= vm_cache_alloc(&handle_table_cache, VM_NORMAL);
+	if (!out) {
+		return NULL;
+	}
+
+	memset(out, 0x0, sizeof *out);
+
+	return out;
+}
+
+void handle_table_destroy(struct handle_table *tab)
+{
+}
+
+static kern_status_t decode_handle_indices(
+	kern_handle_t handle,
+	unsigned int indices[MAX_TABLE_DEPTH])
+{
+	for (int i = 0; i < MAX_TABLE_DEPTH; i++) {
+		unsigned int div = (i > 0 ? REFS_PER_TABLE : HANDLES_PER_TABLE);
+
+		unsigned int v = handle % div;
+		indices[MAX_TABLE_DEPTH - i - 1] = v;
+		handle /= div;
+	}
+
+	return handle == 0 ? KERN_OK : KERN_INVALID_ARGUMENT;
+}
+
+static kern_status_t encode_handle_indices(
+	unsigned int indices[MAX_TABLE_DEPTH],
+	kern_handle_t *out_handle)
+{
+	kern_handle_t handle = 0;
+	unsigned int mul = 1;
+
+	for (int i = MAX_TABLE_DEPTH - 1; i >= 0; i--) {
+		unsigned int v = indices[i] * mul;
+		handle += v;
+		mul *= REFS_PER_TABLE;
+	}
+
+	*out_handle = handle;
+	return KERN_OK;
+}
+
+kern_status_t handle_table_alloc_handle(
+	struct handle_table *tab,
+	struct handle **out_slot,
+	kern_handle_t *out_handle)
+{
+	int i;
+	unsigned int indices[MAX_TABLE_DEPTH] = {0};
+	static const unsigned int reserved_indices[MAX_TABLE_DEPTH] = {0};
+
+	for (i = 0; i < MAX_TABLE_DEPTH - 1; i++) {
+		unsigned int next_index = bitmap_lowest_clear(
+			tab->t_subtables.t_subtable_map,
+			REFS_PER_TABLE);
+
+		if (next_index == BITMAP_NPOS) {
+			return KERN_NO_ENTRY;
+		}
+
+		struct handle_table *next
+			= tab->t_subtables.t_subtable_list[next_index];
+		if (!next) {
+			next = handle_table_create();
+			tab->t_subtables.t_subtable_list[next_index] = next;
+		}
+
+		if (!next) {
+			return KERN_NO_MEMORY;
+		}
+
+		indices[i] = next_index;
+		tab = next;
+	}
+
+	if (memcmp(indices, reserved_indices, sizeof indices) == 0) {
+		bitmap_fill(tab->t_handles.t_handle_map, RESERVED_HANDLES);
+	}
+
+	unsigned int handle_index = bitmap_lowest_clear(
+		tab->t_handles.t_handle_map,
+		HANDLES_PER_TABLE);
+	if (handle_index == BITMAP_NPOS) {
+		return KERN_NO_ENTRY;
+	}
+
+	bitmap_set(tab->t_handles.t_handle_map, handle_index);
+	memset(&tab->t_handles.t_handle_list[handle_index],
+	       0x0,
+	       sizeof(struct handle));
+
+	indices[i] = handle_index;
+
+	*out_slot = &tab->t_handles.t_handle_list[handle_index];
+	return encode_handle_indices(indices, out_handle);
+}
+
+kern_status_t handle_table_free_handle(
+	struct handle_table *tab,
+	kern_handle_t handle)
+{
+	unsigned int indices[MAX_TABLE_DEPTH];
+	if (decode_handle_indices(handle, indices) != KERN_OK) {
+		return KERN_NO_ENTRY;
+	}
+
+	int i;
+	for (i = 0; i < MAX_TABLE_DEPTH - 1; i++) {
+		struct handle_table *next
+			= tab->t_subtables.t_subtable_list[indices[i]];
+		if (!next) {
+			return KERN_NO_ENTRY;
+		}
+
+		bitmap_clear(tab->t_subtables.t_subtable_map, indices[i]);
+		tab = next;
+	}
+
+	unsigned int handle_index = indices[i];
+	if (!bitmap_check(tab->t_handles.t_handle_map, handle_index)) {
+		return KERN_NO_ENTRY;
+	}
+
+	bitmap_clear(tab->t_handles.t_handle_map, handle_index);
+	struct handle *handle_entry
+		= &tab->t_handles.t_handle_list[handle_index];
+
+	if (handle_entry->h_object) {
+		object_remove_handle(handle_entry->h_object);
+	}
+
+	memset(handle_entry, 0x0, sizeof *handle_entry);
+	return KERN_OK;
+}
+
+struct handle *handle_table_get_handle(
+	struct handle_table *tab,
+	kern_handle_t handle)
+{
+	unsigned int indices[MAX_TABLE_DEPTH];
+	if (decode_handle_indices(handle, indices) != KERN_OK) {
+		return NULL;
+	}
+
+	int i;
+	for (i = 0; i < MAX_TABLE_DEPTH - 1; i++) {
+		struct handle_table *next
+			= tab->t_subtables.t_subtable_list[indices[i]];
+		if (!next) {
+			return NULL;
+		}
+
+		tab = next;
+	}
+
+	unsigned int handle_index = indices[i];
+	if (!bitmap_check(tab->t_handles.t_handle_map, handle_index)) {
+		return NULL;
+	}
+
+	if (!tab->t_handles.t_handle_list[handle_index].h_object) {
+		return NULL;
+	}
+
+	return &tab->t_handles.t_handle_list[handle_index];
+}
+
+kern_status_t handle_table_transfer(
+	struct vm_region *dst_region,
+	struct handle_table *dst,
+	kern_msg_handle_t *dst_handles,
+	size_t dst_handles_max,
+	struct vm_region *src_region,
+	struct handle_table *src,
+	kern_msg_handle_t *src_handles,
+	size_t src_handles_count)
+{
+	kern_status_t status = KERN_OK;
+	size_t to_transfer = MIN(dst_handles_max, src_handles_count);
+
+	size_t i = 0;
+	for (size_t i = 0; i < to_transfer; i++) {
+		kern_msg_handle_t src_handle = {0}, dst_handle = {0};
+		virt_addr_t src_handle_addr
+			= (virt_addr_t)src_handles + (i * sizeof src_handle);
+		virt_addr_t dst_handle_addr
+			= (virt_addr_t)dst_handles + (i * sizeof dst_handle);
+		status = vm_region_read_kernel(
+			src_region,
+			src_handle_addr,
+			sizeof src_handle,
+			&src_handle,
+			NULL);
+
+		if (status != KERN_OK) {
+			src_handle.hnd_result = KERN_OK;
+			vm_region_write_kernel(
+				src_region,
+				src_handle_addr,
+				sizeof src_handle,
+				&src_handle,
+				NULL);
+			break;
+		}
+
+		struct handle *src_entry
+			= handle_table_get_handle(src, src_handle.hnd_value);
+		struct handle *dst_entry = NULL;
+		kern_handle_t dst_value = KERN_HANDLE_INVALID;
+
+		if (!src_entry) {
+			status = KERN_INVALID_ARGUMENT;
+			src_handle.hnd_result = KERN_OK;
+			vm_region_write_kernel(
+				src_region,
+				src_handle_addr,
+				sizeof src_handle,
+				&src_handle,
+				NULL);
+			break;
+		}
+
+		switch (src_handle.hnd_mode) {
+		case KERN_MSG_HANDLE_IGNORE:
+			break;
+		case KERN_MSG_HANDLE_MOVE:
+			status = handle_table_alloc_handle(
+				dst,
+				&dst_entry,
+				&dst_value);
+			if (status != KERN_OK) {
+				break;
+			}
+
+			dst_entry->h_object = src_entry->h_object;
+			dst_entry->h_flags = src_entry->h_flags;
+			object_add_handle(dst_entry->h_object);
+
+			handle_table_free_handle(src, src_handles[i].hnd_value);
+
+			dst_handle.hnd_mode = src_handles[i].hnd_mode;
+			dst_handle.hnd_value = dst_value;
+			dst_handle.hnd_result = KERN_OK;
+			break;
+		case KERN_MSG_HANDLE_COPY:
+			status = handle_table_alloc_handle(
+				dst,
+				&dst_entry,
+				&dst_value);
+			if (status != KERN_OK) {
+				break;
+			}
+
+			dst_entry->h_object = src_entry->h_object;
+			dst_entry->h_flags = src_entry->h_flags;
+			object_add_handle(dst_entry->h_object);
+
+			dst_handle.hnd_mode = src_handles[i].hnd_mode;
+			dst_handle.hnd_value = dst_value;
+			dst_handle.hnd_result = KERN_OK;
+			break;
+		default:
+			status = KERN_INVALID_ARGUMENT;
+			break;
+		}
+
+		src_handle.hnd_result = status;
+
+		vm_region_write_kernel(
+			src_region,
+			src_handle_addr,
+			sizeof src_handle,
+			&src_handle,
+			NULL);
+		vm_region_write_kernel(
+			dst_region,
+			dst_handle_addr,
+			sizeof dst_handle,
+			&dst_handle,
+			NULL);
+	}
+
+	for (; i < src_handles_count; i++) {
+		kern_msg_handle_t handle = {0};
+		virt_addr_t handle_addr
+			= (virt_addr_t)src_handles + (i * sizeof handle);
+		vm_region_read_kernel(
+			src_region,
+			handle_addr,
+			sizeof handle,
+			&handle,
+			NULL);
+
+		if (handle.hnd_mode != KERN_MSG_HANDLE_MOVE) {
+			continue;
+		}
+
+		struct handle *src_entry
+			= handle_table_get_handle(src, handle.hnd_value);
+		if (src_entry) {
+			object_remove_handle(src_entry->h_object);
+			handle_table_free_handle(src, handle.hnd_value);
+		}
+	}
+
+	return status;
+}

kernel/iovec.c

@@ -0,0 +1,121 @@
+#include <kernel/iovec.h>
+#include <kernel/libc/string.h>
+#include <kernel/util.h>
+#include <kernel/vm-region.h>
+
+static bool read_iovec(
+	struct iovec_iterator *it,
+	size_t index,
+	kern_iovec_t *out)
+{
+	if (index >= it->it_nr_vecs) {
+		return false;
+	}
+
+	if (!it->it_region) {
+		memcpy(out, &it->it_vecs[index], sizeof *out);
+		return true;
+	}
+
+	size_t nr_read = 0;
+	kern_status_t status = vm_region_read_kernel(
+		it->it_region,
+		(virt_addr_t)it->it_vecs + (index * sizeof(kern_iovec_t)),
+		sizeof(kern_iovec_t),
+		out,
+		&nr_read);
+
+	return (status == KERN_OK && nr_read != sizeof(kern_iovec_t));
+}
+
+void iovec_iterator_begin_user(
+	struct iovec_iterator *it,
+	struct vm_region *region,
+	const kern_iovec_t *vecs,
+	size_t nr_vecs)
+{
+	memset(it, 0x0, sizeof *it);
+	it->it_region = region;
+	it->it_vecs = vecs;
+	it->it_nr_vecs = nr_vecs;
+
+	kern_iovec_t iov;
+
+	while (it->it_vec_ptr < nr_vecs) {
+		read_iovec(it, it->it_vec_ptr, &iov);
+		if (iov.io_len > 0) {
+			break;
+		}
+
+		it->it_vec_ptr++;
+	}
+
+	if (it->it_vec_ptr >= nr_vecs) {
+		return;
+	}
+
+	it->it_base = iov.io_base;
+	it->it_len = iov.io_len;
+}
+
+void iovec_iterator_begin(
+	struct iovec_iterator *it,
+	const kern_iovec_t *vecs,
+	size_t nr_vecs)
+{
+	memset(it, 0x0, sizeof *it);
+	it->it_vecs = vecs;
+	it->it_nr_vecs = nr_vecs;
+
+	while (it->it_vec_ptr < nr_vecs) {
+		if (vecs[it->it_vec_ptr].io_len > 0) {
+			break;
+		}
+
+		it->it_vec_ptr++;
+	}
+
+	if (it->it_vec_ptr >= nr_vecs) {
+		it->it_len = 0;
+		it->it_base = 0;
+		return;
+	}
+
+	it->it_base = vecs[it->it_vec_ptr].io_base;
+	it->it_len = vecs[it->it_vec_ptr].io_len;
+}
+
+void iovec_iterator_seek(struct iovec_iterator *it, size_t nr_bytes)
+{
+	while (nr_bytes > 0) {
+		size_t to_seek = MIN(nr_bytes, it->it_len);
+
+		if (to_seek < it->it_len) {
+			it->it_len -= to_seek;
+			it->it_base += to_seek;
+			break;
+		}
+
+		nr_bytes -= to_seek;
+		kern_iovec_t iov;
+
+		it->it_vec_ptr++;
+		while (it->it_vec_ptr < it->it_nr_vecs) {
+			read_iovec(it, it->it_vec_ptr, &iov);
+			if (iov.io_len > 0) {
+				break;
+			}
+
+			it->it_vec_ptr++;
+		}
+
+		if (it->it_vec_ptr >= it->it_nr_vecs) {
+			it->it_len = 0;
+			it->it_base = 0;
+			return;
+		}
+
+		it->it_base = iov.io_base;
+		it->it_len = iov.io_len;
+	}
+}

kernel/object.c

@@ -1,12 +1,26 @@
-#include <mango/locks.h>
+#include <kernel/locks.h>
-#include <mango/object.h>
+#include <kernel/object.h>
-#include <mango/queue.h>
+#include <kernel/queue.h>
 
 #define HAS_OP(obj, opname) ((obj)->ob_type->ob_ops.opname)
 
 static struct queue object_types;
 static spin_lock_t object_types_lock = SPIN_LOCK_INIT;
 
+static koid_t koid_alloc(void)
+{
+	static koid_t counter = 0;
+	static spin_lock_t lock = SPIN_LOCK_INIT;
+
+	unsigned long flags;
+	spin_lock_irqsave(&lock, &flags);
+	koid_t result = counter;
+	counter++;
+	spin_unlock_irqrestore(&lock, flags);
+
+	return result;
+}
+
 kern_status_t object_bootstrap(void)
 {
 	return KERN_OK;
@@ -21,7 +35,6 @@ kern_status_t object_type_register(struct object_type *p)
 
 	p->ob_cache.c_name = p->ob_name;
 	p->ob_cache.c_obj_size = p->ob_size;
-	p->ob_cache.c_page_order = VM_PAGE_16K;
 
 	vm_cache_init(&p->ob_cache);
 	p->ob_flags |= OBJTYPE_INIT;
@@ -51,11 +64,10 @@ struct object *object_create(struct object_type *type)
 		return NULL;
 	}
 
-	memset(obj_buf, 0x00, type->ob_size);
-
 	struct object *obj = (struct object *)((unsigned char *)obj_buf
 	                                       + type->ob_header_offset);
 
+	obj->ob_id = koid_alloc();
 	obj->ob_type = type;
 	obj->ob_lock = SPIN_LOCK_INIT;
 	obj->ob_magic = OBJECT_MAGIC;
@@ -71,7 +83,49 @@ struct object *object_ref(struct object *obj)
 	return obj;
 }
 
-void object_deref(struct object *obj)
+static void __cleanup(struct object *obj, struct queue *queue)
+{
+	if (HAS_OP(obj, destroy)) {
+		obj->ob_type->ob_ops.destroy(obj, queue);
+	}
+
+	vm_cache_free(&obj->ob_type->ob_cache, obj);
+}
+
+static void object_cleanup(struct object *obj, unsigned long flags)
+{
+	if (obj->ob_refcount > 0 || obj->ob_handles > 0) {
+		spin_unlock_irqrestore(&obj->ob_lock, flags);
+		return;
+	}
+
+	struct queue queue = QUEUE_INIT;
+	__cleanup(obj, &queue);
+
+	if (!HAS_OP(obj, destroy_recurse)) {
+		return;
+	}
+
+	while (!queue_empty(&queue)) {
+		struct queue_entry *entry = queue_pop_front(&queue);
+		struct object *child = NULL;
+		obj->ob_type->ob_ops.destroy_recurse(entry, &child);
+		if (!child) {
+			continue;
+		}
+
+		if (child->ob_refcount > 1) {
+			child->ob_refcount--;
+			continue;
+		}
+
+		if (child->ob_refcount == 0 && child->ob_handles == 0) {
+			__cleanup(child, &queue);
+		}
+	}
+}
+
+void object_unref(struct object *obj)
 {
 	unsigned long flags;
 	spin_lock_irqsave(&obj->ob_lock, &flags);
@@ -82,17 +136,26 @@ void object_deref(struct object *obj)
 	}
 
 	obj->ob_refcount--;
+	object_cleanup(obj, flags);
+}
 
-	if (obj->ob_refcount > 0) {
+void object_add_handle(struct object *obj)
+{
+	obj->ob_handles++;
+}
+
+void object_remove_handle(struct object *obj)
+{
+	unsigned long flags;
+	spin_lock_irqsave(&obj->ob_lock, &flags);
+
+	if (obj->ob_handles == 0) {
 		spin_unlock_irqrestore(&obj->ob_lock, flags);
 		return;
 	}
 
-	if (HAS_OP(obj, destroy)) {
+	obj->ob_handles--;
-		obj->ob_type->ob_ops.destroy(obj);
+	object_cleanup(obj, flags);
-	}
-
-	vm_cache_free(&obj->ob_type->ob_cache, obj);
 }
 
 void object_lock(struct object *obj)
@@ -115,6 +178,38 @@ void object_unlock_irqrestore(struct object *obj, unsigned long flags)
 	spin_unlock_irqrestore(&obj->ob_lock, flags);
 }
 
+void object_lock_pair_irqsave(
+	struct object *a,
+	struct object *b,
+	unsigned long *flags)
+{
+	if (a == b) {
+		object_lock_irqsave(a, flags);
+	} else if (a < b) {
+		object_lock_irqsave(a, flags);
+		object_lock(b);
+	} else {
+		object_lock_irqsave(b, flags);
+		object_lock(a);
+	}
+}
+
+void object_unlock_pair_irqrestore(
+	struct object *a,
+	struct object *b,
+	unsigned long flags)
+{
+	if (a == b) {
+		object_unlock_irqrestore(a, flags);
+	} else if (a < b) {
+		object_unlock(b);
+		object_unlock_irqrestore(a, flags);
+	} else {
+		object_unlock(a);
+		object_unlock_irqrestore(b, flags);
+	}
+}
+
 void *object_data(struct object *obj)
 {
 	return (char *)obj + sizeof *obj;