libmango: add userspace syscall call-gates

the lowest 2 bits of handle values are no longer unused, and 0 is
now a valid handle value.

the first 64 handles are now reserved, and will not be automatically
allocated by the kernel. however, they are still valid handles, and
other handles can be moved to this area using an as-yet-unwritten
function. this is to allow support for standard POSIX file descriptors,
which require the values 0, 1, and 2.

CMakeLists.txt

@@ -12,12 +12,11 @@ set(kernel_arch x86_64)
 set(kernel_name "Mango")
 set(kernel_exe_name "mango_kernel")
 
-set(generic_src_dirs ds init kernel libc sched util vm)
+set(generic_src_dirs ds init kernel libc sched util vm syscall)
 set(kernel_sources "")
 set(kernel_headers "")
 
 foreach (dir ${generic_src_dirs})
-	message(STATUS ${dir})
 	file(GLOB_RECURSE dir_sources ${dir}/*.c)
 	file(GLOB_RECURSE dir_headers ${dir}/*.h)
 
@@ -41,6 +40,7 @@ add_executable(${kernel_exe_name}
 target_include_directories(${kernel_exe_name} PRIVATE
 	include
 	libc/include
+	libmango/include
 	arch/${kernel_arch}/include)
 target_compile_options(${kernel_exe_name} PRIVATE
 	-nostdlib -ffreestanding

arch/user/cpu.c

@@ -1,5 +1,5 @@
 #include <unistd.h>
-#include <mango/machine/cpu.h>
+#include <kernel/machine/cpu.h>
 
 int ml_init_bootcpu(void)
 {

arch/user/hwlock.c

@@ -1,5 +1,5 @@
-#include <mango/machine/hwlock.h>
-#include <mango/compiler.h>
+#include <kernel/machine/hwlock.h>
+#include <kernel/compiler.h>
 
 void ml_hwlock_lock(ml_hwlock_t *lck)
 {

arch/user/include/mango/machine/init.h

@@ -1,6 +1,7 @@
 #ifndef MANGO_X86_64_INIT_H_
 #define MANGO_X86_64_INIT_H_
 
+#include <stddef.h>
 #include <stdint.h>
 
 #ifdef __cplusplus
@@ -12,12 +13,14 @@ extern "C" {
 
 #ifdef __APPLE__
 #define __define_initcall(fn, id)                                              \
-	static initcall_t __initcall_##fn##id __used \
-	__section("__DATA,__initcall" __X(id) ".init") = (fn)
+	static initcall_t __initcall_##fn##id __used __section(                \
+		"__DATA,__initcall" __X(id) ".init")                           \
+		= (fn)
 #else
 #define __define_initcall(fn, id)                                              \
-	static initcall_t __initcall_##fn##id __used \
-	__section("initcall" __X(id) "_init") = (fn)
+	static initcall_t __initcall_##fn##id __used __section(                \
+		"initcall" __X(id) "_init")                                    \
+		= (fn)
 #endif
 
 extern int ml_init(uintptr_t arg);

arch/user/include/mango/machine/pmap.h

@@ -1,6 +1,8 @@
 #ifndef MANGO_USER_PMAP_H_
 #define MANGO_USER_PMAP_H_
 
+#include <stdint.h>
+
 typedef uintptr_t ml_pmap_t;
 typedef uint64_t ml_pfn_t;
 

arch/user/init.c

@@ -1,11 +1,11 @@
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <mango/init.h>
-#include <mango/memblock.h>
-#include <mango/vm.h>
-#include <mango/object.h>
-#include <mango/printk.h>
+#include <kernel/init.h>
+#include <kernel/memblock.h>
+#include <kernel/vm.h>
+#include <kernel/object.h>
+#include <kernel/printk.h>
 #include <arch/stdcon.h>
 #include <sys/mman.h>
 

arch/user/initcall.c

@@ -1,4 +1,4 @@
-#include <mango/init.h>
+#include <kernel/init.h>
 
 #ifdef __APPLE__
 extern char __start_initcall0[] __asm("section$start$__DATA$__initcall0.init");

arch/user/stdcon.c

@@ -1,10 +1,10 @@
-#include <mango/libc/string.h>
-#include <mango/libc/ctype.h>
+#include <kernel/libc/string.h>
+#include <kernel/libc/ctype.h>
 #include <stdint.h>
 #include <stdio.h>
-#include <mango/console.h>
-#include <mango/vm.h>
-#include <mango/printk.h>
+#include <kernel/console.h>
+#include <kernel/vm.h>
+#include <kernel/printk.h>
 
 static void stdcon_write(struct console *con, const char *s, unsigned int len)
 {

arch/x86_64/asm-offset.c

@@ -1,5 +1,5 @@
-#include <mango/sched.h>
-#include <mango/compiler.h>
+#include <kernel/sched.h>
+#include <kernel/compiler.h>
 
 //size_t THREAD_sp = offsetof(struct thread, tr_sp);
 

arch/x86_64/cpu.c

@@ -1,11 +1,14 @@
-#include <mango/machine/cpu.h>
 #include <arch/msr.h>
+#include <kernel/machine/cpu.h>
 
 int ml_cpu_block_init(ml_cpu_block *p)
 {
 	p->c_this = p;
 	gdt_init(&p->c_gdt, &p->c_gdt_ptr);
 	idt_init(&p->c_idt_ptr);
+	tss_init(&p->c_tss, &p->c_tss_ptr);
+
+	gdt_write_tss(&p->c_gdt, &p->c_tss);
 	return 0;
 }
 
@@ -13,6 +16,27 @@ int ml_cpu_block_use(ml_cpu_block *p)
 {
 	gdt_load(&p->c_gdt_ptr);
 	idt_load(&p->c_idt_ptr);
+	tss_load(&p->c_tss);
 	wrmsr(MSR_GS_BASE, (uint64_t)p);
 	return 0;
 }
+
+virt_addr_t ml_cpu_block_get_kstack(ml_cpu_block *p)
+{
+	return tss_get_kstack(&p->c_tss);
+}
+
+virt_addr_t ml_cpu_block_get_ustack(ml_cpu_block *p)
+{
+	return tss_get_ustack(&p->c_tss);
+}
+
+void ml_cpu_block_set_kstack(ml_cpu_block *p, virt_addr_t sp)
+{
+	tss_set_kstack(&p->c_tss, sp);
+}
+
+void ml_cpu_block_set_ustack(ml_cpu_block *p, virt_addr_t sp)
+{
+	tss_set_ustack(&p->c_tss, sp);
+}

arch/x86_64/e820.c

@@ -1,8 +1,8 @@
-#include "mango/types.h"
-#include <mango/memblock.h>
-#include <mango/printk.h>
-#include <mango/util.h>
 #include <arch/e820.h>
+#include <kernel/memblock.h>
+#include <kernel/printk.h>
+#include <kernel/types.h>
+#include <kernel/util.h>
 
 void e820_scan(multiboot_memory_map_t *mmap, size_t len)
 {
@@ -36,7 +36,9 @@ void e820_scan(multiboot_memory_map_t *mmap, size_t len)
 		}
 
 		printk("e820: [mem 0x%016llx-0x%016llx] %s",
-		       entry->addr, entry->addr + entry->len - 1, type);
+		       entry->addr,
+		       entry->addr + entry->len - 1,
+		       type);
 
 		memblock_add(entry->addr, entry->len);
 
@@ -53,7 +55,12 @@ void e820_scan(multiboot_memory_map_t *mmap, size_t len)
 
 	char str_mem_total[64], str_mem_reserved[64];
 	data_size_to_string(mem_total, str_mem_total, sizeof str_mem_total);
-	data_size_to_string(mem_reserved, str_mem_reserved, sizeof str_mem_reserved);
+	data_size_to_string(
+		mem_reserved,
+		str_mem_reserved,
+		sizeof str_mem_reserved);
 
-	printk("e820: total memory: %s, hw reserved: %s", str_mem_total, str_mem_reserved);
+	printk("e820: total memory: %s, hw reserved: %s",
+	       str_mem_total,
+	       str_mem_reserved);
 }

arch/x86_64/gdt.c

@@ -1,5 +1,8 @@
-#include <mango/libc/string.h>
 #include <arch/gdt.h>
+#include <arch/tss.h>
+#include <kernel/libc/string.h>
+#include <kernel/types.h>
+#include <stddef.h>
 
 static void init_entry(struct gdt_entry *entry, int access, int flags)
 {
@@ -15,13 +18,42 @@ int gdt_init(struct gdt *gdt, struct gdt_ptr *gdtp)
 {
 	memset(&gdt->g_entries[0], 0x0, sizeof gdt->g_entries[0]);
 
-	init_entry(&gdt->g_entries[1], GDT_A_PRESENT | GDT_A_CODEREAD | GDT_A_CODE, GDT_F_64BIT);
-	init_entry(&gdt->g_entries[2], GDT_A_PRESENT | GDT_A_DATAWRITE | GDT_A_DATA, GDT_F_64BIT);
-	init_entry(&gdt->g_entries[3], GDT_A_PRESENT | GDT_A_USER | GDT_A_CODEREAD | GDT_A_CODE, GDT_F_64BIT);
-	init_entry(&gdt->g_entries[4], GDT_A_PRESENT | GDT_A_USER | GDT_A_DATAWRITE | GDT_A_DATA, GDT_F_64BIT);
+	init_entry(
+		&gdt->g_entries[1],
+		GDT_A_PRESENT | GDT_A_CODEREAD | GDT_A_CODE,
+		GDT_F_64BIT);
+	init_entry(
+		&gdt->g_entries[2],
+		GDT_A_PRESENT | GDT_A_DATAWRITE | GDT_A_DATA,
+		GDT_F_64BIT);
+	init_entry(
+		&gdt->g_entries[3],
+		GDT_A_PRESENT | GDT_A_USER | GDT_A_DATAWRITE | GDT_A_DATA,
+		GDT_F_64BIT);
+	init_entry(
+		&gdt->g_entries[4],
+		GDT_A_PRESENT | GDT_A_USER | GDT_A_CODEREAD | GDT_A_CODE,
+		GDT_F_64BIT);
 
 	gdtp->g_ptr = (uint64_t)gdt;
 	gdtp->g_limit = sizeof(*gdt) - 1;
 
 	return 0;
 }
+
+void gdt_write_tss(struct gdt *gdt, struct tss *tss)
+{
+	struct tss_gdt_entry *tss_entry = &gdt->g_tss;
+
+	virt_addr_t base = (virt_addr_t)tss;
+	size_t limit = sizeof *tss;
+
+	tss_entry->ge_base_low = (base & 0xFFFF);
+	tss_entry->ge_base_mid = (base >> 16) & 0xFF;
+	tss_entry->ge_base_hi = (base >> 24) & 0xFF;
+	tss_entry->ge_base_ext = (base >> 32) & 0xFFFFFFFF;
+	tss_entry->ge_limit_low = (limit & 0xFFFF);
+	tss_entry->ge_gran = (limit >> 16) & 0xF;
+	tss_entry->ge_access = 0xE9;
+	tss_entry->ge_reserved = 0;
+}

arch/x86_64/include/arch/gdt.h

@@ -1,7 +1,8 @@
 #ifndef ARCH_GDT_H_
 #define ARCH_GDT_H_
 
-#include <mango/compiler.h>
+#include <arch/tss.h>
+#include <kernel/compiler.h>
 #include <stdint.h>
 
 #ifdef __cplusplus
@@ -33,6 +34,7 @@ struct gdt_entry {
 
 struct gdt {
 	struct gdt_entry g_entries[NR_GDT_ENTRIES];
+	struct tss_gdt_entry g_tss;
 } __packed;
 
 struct gdt_ptr {
@@ -41,6 +43,7 @@ struct gdt_ptr {
 } __packed;
 
 extern int gdt_init(struct gdt *gdt, struct gdt_ptr *gdtp);
+extern void gdt_write_tss(struct gdt *gdt, struct tss *tss);
 extern int gdt_load(struct gdt_ptr *gdtp);
 
 #ifdef __cplusplus

arch/x86_64/include/arch/irq.h

@@ -1,8 +1,8 @@
 #ifndef ARCH_IRQ_H_
 #define ARCH_IRQ_H_
 
-#include <mango/compiler.h>
-#include <mango/queue.h>
+#include <kernel/compiler.h>
+#include <kernel/queue.h>
 #include <stdint.h>
 
 #ifdef __cplusplus
@@ -11,6 +11,8 @@ extern "C" {
 
 #define NR_IDT_ENTRIES 256
 
+struct ml_cpu_context;
+
 enum irq_vector {
 	IRQ0 = 32,
 	IRQ1,
@@ -35,13 +37,6 @@ struct irq_hook {
 	int (*irq_callback)(void);
 };
 
-struct cpu_context {
-	uint64_t r15, r14, r13, r12, r11, r10, r9, r8;
-	uint64_t rdi, rsi, rbp, unused_rsp, rbx, rdx, rcx, rax;
-	uint64_t int_no, err_no;
-	uint64_t rip, cs, rflags, rsp, ss;
-} __packed;
-
 struct idt_entry {
 	uint16_t base_low;
 	uint16_t selector;
@@ -64,7 +59,7 @@ struct idt_ptr {
 	uintptr_t i_base;
 } __packed;
 
-typedef void (*int_hook)(struct cpu_context *);
+typedef void (*int_hook)(struct ml_cpu_context *);
 
 extern int idt_init(struct idt_ptr *idtp);
 extern int idt_load(struct idt_ptr *idtp);

arch/x86_64/include/arch/paging.h

@@ -1,8 +1,8 @@
 #ifndef ARCH_PAGING_H_
 #define ARCH_PAGING_H_
 
-#include <mango/types.h>
-#include <mango/compiler.h>
+#include <kernel/types.h>
+#include <kernel/compiler.h>
 
 #ifdef __cplusplus
 extern "C" {

arch/x86_64/include/arch/tss.h

@@ -0,0 +1,55 @@
+#ifndef ARCH_TSS_H_
+#define ARCH_TSS_H_
+
+#include <kernel/compiler.h>
+#include <kernel/types.h>
+#include <stdint.h>
+
+#define TSS_GDT_INDEX 5
+
+struct tss {
+	uint32_t res0;
+	uint64_t rsp0;
+	uint64_t rsp1;
+	uint64_t rsp2;
+	uint64_t res1;
+	uint64_t ist1;
+	uint64_t ist2;
+	uint64_t ist3;
+	uint64_t ist4;
+	uint64_t ist5;
+	uint64_t ist6;
+	uint64_t ist7;
+	uint64_t res2;
+	uint16_t res3;
+	uint16_t iomap_offset;
+} __packed;
+
+struct tss_gdt_entry {
+	/* these fields are copied from struct gdt_entry */
+	uint16_t ge_limit_low;
+	uint16_t ge_base_low;
+	uint8_t ge_base_mid;
+	uint8_t ge_access;
+	uint8_t ge_gran;
+	uint8_t ge_base_hi;
+	/* these fields are specific to the TSS entry */
+	uint32_t ge_base_ext;
+	uint32_t ge_reserved;
+} __packed;
+
+struct tss_ptr {
+	uint16_t tss_limit;
+	uint64_t tss_base;
+} __packed;
+
+extern void tss_init(struct tss *tss, struct tss_ptr *ptr);
+extern void tss_load(struct tss *tss);
+
+extern virt_addr_t tss_get_kstack(struct tss *tss);
+extern virt_addr_t tss_get_ustack(struct tss *tss);
+
+extern void tss_set_kstack(struct tss *tss, virt_addr_t sp);
+extern void tss_set_ustack(struct tss *tss, virt_addr_t sp);
+
+#endif

arch/x86_64/include/kernel/machine/cpu.h

@@ -0,0 +1,77 @@
+#ifndef KERNEL_X86_64_CPU_H_
+#define KERNEL_X86_64_CPU_H_
+
+#include <arch/gdt.h>
+#include <arch/irq.h>
+#include <arch/tss.h>
+#include <kernel/types.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define ML_BIG_ENDIAN            0
+
+#define ml_cpu_block_get_id(p)   ((p)->c_cpu_id)
+#define ml_cpu_block_get_data(p) ((p)->c_data)
+
+#if 0
+#define ml_read_sp(sp, bp)                                                     \
+	asm volatile("mov %%rsp, %0" : "=r"(sp));                              \
+	asm volatile("mov %%rbp, %0" : "=r"(bp));
+#endif
+
+struct cpu_data;
+
+typedef struct ml_cpu_block {
+	struct ml_cpu_block *c_this;
+
+	struct gdt c_gdt;
+	struct gdt_ptr c_gdt_ptr;
+
+	struct tss c_tss;
+	struct tss_ptr c_tss_ptr;
+
+	struct idt_ptr c_idt_ptr;
+	unsigned int c_cpu_id;
+
+	struct cpu_data *c_data;
+} ml_cpu_block;
+
+struct ml_int_context {
+	uint64_t rip, cs, rflags, rsp, ss;
+};
+
+struct ml_cpu_context {
+	uint64_t r15, r14, r13, r12, r11, r10, r9, r8;
+	uint64_t rdi, rsi, rbp, unused_rsp, rbx, rdx, rcx, rax;
+	uint64_t int_no, err_no;
+	uint64_t rip, cs, rflags, rsp, ss;
+} __packed;
+
+#define ml_cpu_pause()   __asm__ __volatile__("hlt")
+#define ml_cpu_relax()   __asm__ __volatile__("pause")
+
+#define ml_int_disable() __asm__ __volatile__("cli")
+#define ml_int_enable()  __asm__ __volatile__("sti")
+
+extern int ml_init_bootcpu(void);
+
+extern int ml_cpu_block_init(ml_cpu_block *p);
+extern int ml_cpu_block_use(ml_cpu_block *p);
+
+extern virt_addr_t ml_cpu_block_get_ustack(ml_cpu_block *p);
+extern virt_addr_t ml_cpu_block_get_kstack(ml_cpu_block *p);
+
+extern void ml_cpu_block_set_ustack(ml_cpu_block *p, virt_addr_t sp);
+extern void ml_cpu_block_set_kstack(ml_cpu_block *p, virt_addr_t sp);
+
+/* defined in cpu_ctrl.S */
+extern void ml_halt_cpu(void);
+extern ml_cpu_block *ml_this_cpu(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

arch/x86_64/include/kernel/machine/hwlock.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_X86_64_HWLOCK_H_
-#define MANGO_X86_64_HWLOCK_H_
+#ifndef KERNEL_X86_64_HWLOCK_H_
+#define KERNEL_X86_64_HWLOCK_H_
 
 #define ML_HWLOCK_INIT (0)
 

arch/x86_64/include/kernel/machine/init.h

@@ -0,0 +1,28 @@
+#ifndef KERNEL_X86_64_INIT_H_
+#define KERNEL_X86_64_INIT_H_
+
+#include <stddef.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define __X2(x) #x
+#define __X(x)  __X2(x)
+
+#define __define_initcall(fn, id)                                              \
+	static initcall_t __initcall_##fn##id __used __section(                \
+		".initcall" __X(id) ".init")                                   \
+		= (fn)
+
+extern int ml_init(uintptr_t arg);
+
+extern const struct framebuffer_varinfo *bootfb_varinfo(void);
+extern const struct framebuffer_fixedinfo *bootfb_fixedinfo(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

arch/x86_64/include/kernel/machine/irq.h

@@ -0,0 +1,5 @@
+#ifndef KERNEL_X86_64_IRQ_H_
+#define KERNEL_X86_64_IRQ_H_
+
+
+#endif

arch/x86_64/include/kernel/machine/panic.h

@@ -0,0 +1,12 @@
+#ifndef KERNEL_X86_64_PANIC_H_
+#define KERNEL_X86_64_PANIC_H_
+
+#include <stdint.h>
+
+struct ml_cpu_context;
+
+extern void ml_print_cpu_state(struct ml_cpu_context *ctx);
+extern void ml_print_stack_trace(uintptr_t ip);
+extern void ml_print_stack_trace_irq(struct ml_cpu_context *ctx);
+
+#endif

arch/x86_64/include/kernel/machine/pmap.h

@@ -0,0 +1,11 @@
+#ifndef KERNEL_X86_64_PMAP_H_
+#define KERNEL_X86_64_PMAP_H_
+
+#include <arch/paging.h>
+
+#define ML_PMAP_INVALID ((uintptr_t)-1)
+
+typedef pml4t_ptr_t ml_pmap_t;
+typedef uint64_t ml_pfn_t;
+
+#endif

arch/x86_64/include/kernel/machine/thread.h

@@ -0,0 +1,31 @@
+#ifndef KERNEL_X86_64_THREAD_H_
+#define KERNEL_X86_64_THREAD_H_
+
+#include <kernel/sched.h>
+
+struct ml_cpu_context;
+
+/* switch from one thread to another. the stack of the `to` thread must have
+ * been prepared in one of two ways:
+ *  1) a previous call to ml_thread_switch where it was the `from` thread.
+ *  2) a call to ml_thread_prepare_kernel_context
+ * the switch occurs entirely with kernel mode. a further return from an
+ * interrupt context is then used to return to usermode if necessary.
+ */
+extern void ml_thread_switch(struct thread *from, struct thread *to);
+
+/* perform the initial transition to userspace. the stack must be prepared using
+ * ml_thread_prepare_user_context before this function can be used */
+extern void ml_thread_switch_user(void);
+/* prepare the stack so that ml_thread_switch can jump to the specified IP/SP */
+extern void ml_thread_prepare_kernel_context(uintptr_t ip, uintptr_t *sp);
+/* prepare the stack so that ml_thread_switch_user can jump to usermode
+ * with the specified IP/user SP */
+extern kern_status_t ml_thread_prepare_user_context(
+	virt_addr_t ip,
+	virt_addr_t user_sp,
+	virt_addr_t *kernel_sp,
+	const uintptr_t *args,
+	size_t nr_args);
+
+#endif

arch/x86_64/include/kernel/machine/vm.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_X86_64_VM_H_
-#define MANGO_X86_64_VM_H_
+#ifndef KERNEL_X86_64_VM_H_
+#define KERNEL_X86_64_VM_H_
 
 /* kernel higher-half base virtual address. */
 #define VM_KERNEL_VOFFSET 0xFFFFFFFF80000000
@@ -12,7 +12,7 @@
 #define VM_PAGEMAP_LIMIT  0xFFFFC87FFFFFFFFF
 
 #define VM_PAGE_SIZE      0x1000
-#define VM_PAGE_MASK  (VM_PAGE_SIZE-1)
+#define VM_PAGE_MASK      (VM_PAGE_SIZE - 1)
 #define VM_PAGE_SHIFT     12
 
 #define VM_PAGE_MIN_ORDER VM_PAGE_4K
@@ -21,4 +21,10 @@
 #define VM_ZONE_MIN       VM_ZONE_DMA
 #define VM_ZONE_MAX       VM_ZONE_NORMAL
 
+#define VM_USER_BASE      0x0000000000100000
+#define VM_USER_LIMIT     0x00007fffffffffff
+
+#define VM_KERNEL_BASE    0XFFFF800000000000
+#define VM_KERNEL_LIMIT   0XFFFFFFFFFFFFFFFF
+
 #endif

arch/x86_64/include/mango/machine/cpu.h

@@ -1,49 +0,0 @@
-#ifndef MANGO_X86_64_CPU_H_
-#define MANGO_X86_64_CPU_H_
-
-#include <arch/gdt.h>
-#include <arch/irq.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define ML_BIG_ENDIAN 0
-
-#define ml_cpu_block_get_id(p) ((p)->c_cpu_id)
-#define ml_cpu_block_get_data(p) ((p)->c_data)
-
-struct cpu_data;
-
-typedef struct ml_cpu_block {
-	struct ml_cpu_block *c_this;
-
-	struct gdt c_gdt;
-	struct gdt_ptr c_gdt_ptr;
-
-	struct idt_ptr c_idt_ptr;
-	unsigned int c_cpu_id;
-
-	struct cpu_data *c_data;
-} ml_cpu_block;
-
-#define ml_cpu_pause() __asm__ __volatile__("hlt")
-#define ml_cpu_relax() __asm__ __volatile__("pause")
-
-#define ml_int_disable() __asm__ __volatile__("cli")
-#define ml_int_enable() __asm__ __volatile__("sti")
-
-extern int ml_init_bootcpu(void);
-
-extern int ml_cpu_block_init(ml_cpu_block *p);
-extern int ml_cpu_block_use(ml_cpu_block *p);
-
-/* defined in cpu_ctrl.S */
-extern void ml_halt_cpu(void);
-extern ml_cpu_block *ml_this_cpu(void);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

arch/x86_64/include/mango/machine/init.h

@@ -1,26 +0,0 @@
-#ifndef MANGO_X86_64_INIT_H_
-#define MANGO_X86_64_INIT_H_
-
-#include <stdint.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define __X2(x) #x
-#define __X(x) __X2(x)
-
-#define __define_initcall(fn, id) \
-	static initcall_t __initcall_##fn##id __used \
-	__section(".initcall" __X(id) ".init") = (fn)
-
-extern int ml_init(uintptr_t arg);
-
-extern const struct framebuffer_varinfo *bootfb_varinfo(void);
-extern const struct framebuffer_fixedinfo *bootfb_fixedinfo(void);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

arch/x86_64/include/mango/machine/irq.h

@@ -1,5 +0,0 @@
-#ifndef MANGO_X86_64_IRQ_H_
-#define MANGO_X86_64_IRQ_H_
-
-
-#endif

arch/x86_64/include/mango/machine/panic.h

@@ -1,12 +0,0 @@
-#ifndef MANGO_X86_64_PANIC_H_
-#define MANGO_X86_64_PANIC_H_
-
-#include <stdint.h>
-
-struct cpu_context;
-
-extern void ml_print_cpu_state(struct cpu_context *ctx);
-extern void ml_print_stack_trace(uintptr_t ip);
-extern void ml_print_stack_trace_irq(struct cpu_context *ctx);
-
-#endif

arch/x86_64/include/mango/machine/pmap.h

@@ -1,9 +0,0 @@
-#ifndef MANGO_X86_64_PMAP_H_
-#define MANGO_X86_64_PMAP_H_
-
-#include <arch/paging.h>
-
-typedef pml4t_ptr_t ml_pmap_t;
-typedef uint64_t ml_pfn_t;
-
-#endif

arch/x86_64/include/mango/machine/thread.h

@@ -1,10 +0,0 @@
-#ifndef MANGO_X86_64_THREAD_H_
-#define MANGO_X86_64_THREAD_H_
-
-#include <mango/sched.h>
-
-extern void switch_to(struct thread *from, struct thread *to);
-extern void prepare_stack(uintptr_t ip, uintptr_t *sp);
-extern void user_jump(uintptr_t ip, uintptr_t sp);
-
-#endif

arch/x86_64/init.c

@@ -2,20 +2,21 @@
 #include <arch/pit.h>
 #include <arch/serial.h>
 #include <arch/vgacon.h>
-#include <mango/arg.h>
-#include <mango/clock.h>
-#include <mango/console.h>
-#include <mango/cpu.h>
-#include <mango/init.h>
-#include <mango/libc/stdio.h>
-#include <mango/machine/cpu.h>
-#include <mango/memblock.h>
-#include <mango/object.h>
-#include <mango/percpu.h>
-#include <mango/pmap.h>
-#include <mango/printk.h>
-#include <mango/types.h>
-#include <mango/vm.h>
+#include <kernel/arg.h>
+#include <kernel/bsp.h>
+#include <kernel/clock.h>
+#include <kernel/console.h>
+#include <kernel/cpu.h>
+#include <kernel/init.h>
+#include <kernel/libc/stdio.h>
+#include <kernel/machine/cpu.h>
+#include <kernel/memblock.h>
+#include <kernel/object.h>
+#include <kernel/percpu.h>
+#include <kernel/pmap.h>
+#include <kernel/printk.h>
+#include <kernel/types.h>
+#include <kernel/vm.h>
 
 #define PTR32(x) ((void *)((uintptr_t)(x)))
 
@@ -30,7 +31,7 @@ static void bootstrap_cpu_init(void)
 	ml_cpu_block_use(&g_bootstrap_cpu);
 }
 
-static void early_vm_init(void)
+static void early_vm_init(uintptr_t reserve_end)
 {
 	uintptr_t alloc_start = VM_KERNEL_VOFFSET;
 	/* boot code mapped 2 GiB of memory from
@@ -42,10 +43,10 @@ static void early_vm_init(void)
 	       alloc_start,
 	       alloc_end);
 
-	memblock_reserve(0x00, (uintptr_t)__pend);
+	memblock_reserve(0x00, reserve_end);
 	printk("memblock: reserved bios+kernel at [0x%016llx-0x%016llx]",
 	       0,
-	       (uintptr_t)__pend);
+	       reserve_end);
 }
 
 void early_console_init(void)
@@ -70,6 +71,23 @@ static void use_uniprocessor_topology(void)
 	cpu_set_online(0);
 }
 
+static void find_bsp(multiboot_info_t *mb, struct boot_module *out)
+{
+	memset(out, 0x0, sizeof *out);
+
+	printk("modules=%u: %llx", mb->mods_count, mb->mods_addr);
+
+	multiboot_module_t *mods = PTR32(mb->mods_addr);
+	size_t nr_mods = mb->mods_count;
+
+	if (nr_mods < 1) {
+		return;
+	}
+
+	out->mod_base = mods[0].mod_start;
+	out->mod_size = mods[0].mod_end - mods[0].mod_start;
+}
+
 int ml_init(uintptr_t arg)
 {
 	multiboot_info_t *mb = (multiboot_info_t *)arg;
@@ -83,7 +101,16 @@ int ml_init(uintptr_t arg)
 
 	print_kernel_banner();
 
-	early_vm_init();
+	struct boot_module bsp;
+	find_bsp(mb, &bsp);
+	bsp_set_location(&bsp);
+
+	uintptr_t reserve_end = (uintptr_t)__pend;
+	if (bsp.mod_base + bsp.mod_size > reserve_end) {
+		reserve_end = bsp.mod_base + bsp.mod_size;
+	}
+
+	early_vm_init(reserve_end);
 
 	e820_scan(PTR32(mb->mmap_addr), mb->mmap_length);
 
@@ -102,16 +129,20 @@ int ml_init(uintptr_t arg)
 	put_cpu(this_cpu);
 
 	struct vm_zone_descriptor vm_zones[] = {
-		{.zd_id = VM_ZONE_DMA,
+		{
+			.zd_id = VM_ZONE_DMA,
 			.zd_node = 0,
 			.zd_name = "dma",
 			.zd_base = 0x00,
-	         .zd_limit = 0xffffff},
-		{.zd_id = VM_ZONE_NORMAL,
+			.zd_limit = 0xffffff,
+		},
+		{
+			.zd_id = VM_ZONE_NORMAL,
 			.zd_node = 0,
 			.zd_name = "normal",
 			.zd_base = 0x1000000,
-	         .zd_limit = UINTPTR_MAX},
+			.zd_limit = UINTPTR_MAX,
+		},
 	};
 
 	vm_bootstrap(vm_zones, sizeof vm_zones / sizeof vm_zones[0]);

arch/x86_64/initcall.c

@@ -1,4 +1,4 @@
-#include <mango/init.h>
+#include <kernel/init.h>
 
 extern char __initcall0_start[];
 extern char __initcall1_start[];

arch/x86_64/irq.c

@@ -1,15 +1,23 @@
 #include <arch/irq.h>
+#include <arch/msr.h>
 #include <arch/ports.h>
-#include <mango/cpu.h>
-#include <mango/libc/string.h>
-#include <mango/machine/cpu.h>
-#include <mango/machine/irq.h>
-#include <mango/panic.h>
-#include <mango/sched.h>
+#include <kernel/cpu.h>
+#include <kernel/libc/string.h>
+#include <kernel/machine/cpu.h>
+#include <kernel/machine/irq.h>
+#include <kernel/panic.h>
+#include <kernel/sched.h>
+#include <kernel/syscall.h>
 #include <stddef.h>
 
 #define MAX_ISR_HANDLERS  16
 
+#define PF_PRESENT        0x01u
+#define PF_WRITE          0x02u
+#define PF_USER           0x04u
+#define PF_RESERVED_WRITE 0x08u
+#define PF_IFETCH         0x10u
+
 extern void syscall_gate(void);
 extern uintptr_t pf_faultptr(void);
 
@@ -20,6 +28,27 @@ static struct idt idt;
 static int idt_initialised = 0;
 static uintptr_t int_entry_points[NR_IDT_ENTRIES];
 
+static void set_syscall_gate(uintptr_t rip)
+{
+	uint64_t user_cs = 0x13;
+	uint64_t kernel_cs = 0x08;
+
+	uintptr_t star_reg = 0xC0000081;
+	uintptr_t lstar_reg = 0xC0000082;
+	uintptr_t sfmask_reg = 0xC0000084;
+
+	uint64_t selectors = 0;
+	selectors |= (user_cs) << 48;
+	selectors |= (kernel_cs) << 32;
+
+	/* disable interrupts */
+	uint64_t flag_mask = 0x200;
+
+	wrmsr(star_reg, selectors);
+	wrmsr(lstar_reg, rip);
+	wrmsr(sfmask_reg, flag_mask);
+}
+
 static void set_idt_gate(
 	struct idt *idt,
 	uint8_t index,
@@ -39,7 +68,7 @@ static void set_idt_gate(
 	idt->i_entries[index].reserved = 0;
 }
 
-static void gpf_handler(struct cpu_context *regs)
+static void gpf_handler(struct ml_cpu_context *regs)
 {
 	int ext = regs->err_no & 1;
 	int table = (regs->err_no >> 1) & 0x03;
@@ -55,43 +84,33 @@ static void gpf_handler(struct cpu_context *regs)
 		regs->rip);
 }
 
-static void pf_handler(struct cpu_context *regs)
+static void pf_handler(struct ml_cpu_context *regs)
 {
+	enum pmap_fault_flags fault_flags = 0;
+
+	(regs->err_no & PF_PRESENT) && (fault_flags |= PMAP_FAULT_PRESENT);
+	(regs->err_no & PF_USER) && (fault_flags |= PMAP_FAULT_USER);
+	(regs->err_no & PF_WRITE) && (fault_flags |= PMAP_FAULT_WRITE);
+	(regs->err_no & PF_IFETCH) && (fault_flags |= PMAP_FAULT_IFETCH);
+	(regs->err_no & PF_RESERVED_WRITE)
+		&& (fault_flags |= PMAP_FAULT_BADCFG);
+
+	virt_addr_t fault_ptr = pf_faultptr();
+
+	kern_status_t status = pmap_handle_fault(fault_ptr, fault_flags);
+
+	if (status == KERN_OK) {
+		return;
+	}
+
 	panic_irq(
 		regs,
 		"page fault (%016llx %016llx %016llx)",
-		pf_faultptr(),
+		fault_ptr,
 		regs->rip,
 		regs->err_no);
 }
 
-#if 0
-static void set_syscall_gate(uintptr_t rip)
-{
-	/* sysret adds 0x10 to this to get cs, and 0x8 to get ss
-	 * note that the CPU should force the RPL to 3 when loading
-	 * the selector by using user_cs | 3. However, this doesn't happen
-	 * in certain scenarios (specifically, QEMU + KVM on a Ryzen 5 1600X). */
-	uint64_t user_cs = 0x13;
-	uint64_t kernel_cs = 0x8;
-
-	uintptr_t star_reg = 0xC0000081;
-	uintptr_t lstar_reg = 0xC0000082;
-	uintptr_t sfmask_reg = 0xC0000084;
-
-	uint64_t selectors = 0;
-	selectors |= (user_cs) << 48;
-	selectors |= (kernel_cs) << 32;
-
-	/* disable interrupts */
-	uint64_t flag_mask = 0x200;
-
-	write_msr(star_reg, selectors);
-	write_msr(lstar_reg, rip);
-	write_msr(sfmask_reg, flag_mask);
-}
-#endif
-
 static void init_pic(void)
 {
 	// Remap the PIC
@@ -129,6 +148,8 @@ int idt_init(struct idt_ptr *ptr)
 		init_global_idt();
 	}
 
+	set_syscall_gate((uintptr_t)syscall_gate);
+
 	ptr->i_limit = sizeof(idt) - 1;
 	ptr->i_base = (uintptr_t)&idt;
 
@@ -141,7 +162,7 @@ int idt_load(struct idt_ptr *ptr)
 	return 0;
 }
 
-void isr_dispatch(struct cpu_context *regs)
+void isr_dispatch(struct ml_cpu_context *regs)
 {
 	int_hook h = isr_handlers[regs->int_no];
 	if (h) {
@@ -160,7 +181,7 @@ void irq_ack(unsigned int vec)
 	outportb(0x20, 0x20);
 }
 
-void irq_dispatch(struct cpu_context *regs)
+void irq_dispatch(struct ml_cpu_context *regs)
 {
 	end_charge_period();
 
@@ -178,8 +199,40 @@ void irq_dispatch(struct cpu_context *regs)
 	start_charge_period();
 }
 
-void syscall_dispatch(struct cpu_context *regs)
+void syscall_dispatch(struct ml_cpu_context *regs)
 {
+	unsigned int sysid = regs->rax;
+	virt_addr_t syscall_impl = syscall_get_function(sysid);
+
+	if (syscall_impl == 0) {
+		regs->rax = KERN_UNSUPPORTED;
+		return;
+	}
+
+#define SYSCALL_SIGNATURE(...)                                                 \
+	intptr_t (*__VA_ARGS__)(                                               \
+		uintptr_t,                                                     \
+		uintptr_t,                                                     \
+		uintptr_t,                                                     \
+		uintptr_t,                                                     \
+		uintptr_t,                                                     \
+		uintptr_t,                                                     \
+		uintptr_t,                                                     \
+		uintptr_t)
+
+	SYSCALL_SIGNATURE(fn) = (SYSCALL_SIGNATURE())syscall_impl;
+
+	ml_int_enable();
+	regs->rax
+		= fn(regs->rdi,
+	             regs->rsi,
+	             regs->rdx,
+	             regs->r12,
+	             regs->r8,
+	             regs->r9,
+	             regs->r13,
+	             regs->r14);
+	ml_int_disable();
 }
 
 void hook_irq(enum irq_vector vec, struct irq_hook *hook)
@@ -194,263 +247,263 @@ void unhook_irq(enum irq_vector vec, struct irq_hook *hook)
 	queue_delete(hook_queue, &hook->irq_entry);
 }
 
-extern void _isr0();
-extern void _isr1();
-extern void _isr2();
-extern void _isr3();
-extern void _isr4();
-extern void _isr5();
-extern void _isr6();
-extern void _isr7();
-extern void _isr8();
-extern void _isr9();
-extern void _isr10();
-extern void _isr11();
-extern void _isr12();
-extern void _isr13();
-extern void _isr14();
-extern void _isr15();
-extern void _isr16();
-extern void _isr17();
-extern void _isr18();
-extern void _isr19();
-extern void _isr20();
-extern void _isr21();
-extern void _isr22();
-extern void _isr23();
-extern void _isr24();
-extern void _isr25();
-extern void _isr26();
-extern void _isr27();
-extern void _isr28();
-extern void _isr29();
-extern void _isr30();
-extern void _isr31();
+extern void _isr0(void);
+extern void _isr1(void);
+extern void _isr2(void);
+extern void _isr3(void);
+extern void _isr4(void);
+extern void _isr5(void);
+extern void _isr6(void);
+extern void _isr7(void);
+extern void _isr8(void);
+extern void _isr9(void);
+extern void _isr10(void);
+extern void _isr11(void);
+extern void _isr12(void);
+extern void _isr13(void);
+extern void _isr14(void);
+extern void _isr15(void);
+extern void _isr16(void);
+extern void _isr17(void);
+extern void _isr18(void);
+extern void _isr19(void);
+extern void _isr20(void);
+extern void _isr21(void);
+extern void _isr22(void);
+extern void _isr23(void);
+extern void _isr24(void);
+extern void _isr25(void);
+extern void _isr26(void);
+extern void _isr27(void);
+extern void _isr28(void);
+extern void _isr29(void);
+extern void _isr30(void);
+extern void _isr31(void);
 
-extern void _irq0();
-extern void _irq1();
-extern void _irq2();
-extern void _irq3();
-extern void _irq4();
-extern void _irq5();
-extern void _irq6();
-extern void _irq7();
-extern void _irq8();
-extern void _irq9();
-extern void _irq10();
-extern void _irq11();
-extern void _irq12();
-extern void _irq13();
-extern void _irq14();
-extern void _irq15();
-extern void _irq16();
-extern void _irq17();
-extern void _irq18();
-extern void _irq19();
-extern void _irq20();
-extern void _irq21();
-extern void _irq22();
-extern void _irq23();
-extern void _irq24();
-extern void _irq25();
-extern void _irq26();
-extern void _irq27();
-extern void _irq28();
-extern void _irq29();
-extern void _irq30();
-extern void _irq31();
-extern void _irq32();
-extern void _irq33();
-extern void _irq34();
-extern void _irq35();
-extern void _irq36();
-extern void _irq37();
-extern void _irq38();
-extern void _irq39();
-extern void _irq40();
-extern void _irq41();
-extern void _irq42();
-extern void _irq43();
-extern void _irq44();
-extern void _irq45();
-extern void _irq46();
-extern void _irq47();
-extern void _irq48();
-extern void _irq49();
-extern void _irq50();
-extern void _irq51();
-extern void _irq52();
-extern void _irq53();
-extern void _irq54();
-extern void _irq55();
-extern void _irq56();
-extern void _irq57();
-extern void _irq58();
-extern void _irq59();
-extern void _irq60();
-extern void _irq61();
-extern void _irq62();
-extern void _irq63();
-extern void _irq64();
-extern void _irq65();
-extern void _irq66();
-extern void _irq67();
-extern void _irq68();
-extern void _irq69();
-extern void _irq70();
-extern void _irq71();
-extern void _irq72();
-extern void _irq73();
-extern void _irq74();
-extern void _irq75();
-extern void _irq76();
-extern void _irq77();
-extern void _irq78();
-extern void _irq79();
-extern void _irq80();
-extern void _irq81();
-extern void _irq82();
-extern void _irq83();
-extern void _irq84();
-extern void _irq85();
-extern void _irq86();
-extern void _irq87();
-extern void _irq88();
-extern void _irq89();
-extern void _irq90();
-extern void _irq91();
-extern void _irq92();
-extern void _irq93();
-extern void _irq94();
-extern void _irq95();
-extern void _irq96();
-extern void _irq97();
-extern void _irq98();
-extern void _irq99();
-extern void _irq100();
-extern void _irq101();
-extern void _irq102();
-extern void _irq103();
-extern void _irq104();
-extern void _irq105();
-extern void _irq106();
-extern void _irq107();
-extern void _irq108();
-extern void _irq109();
-extern void _irq110();
-extern void _irq111();
-extern void _irq112();
-extern void _irq113();
-extern void _irq114();
-extern void _irq115();
-extern void _irq116();
-extern void _irq117();
-extern void _irq118();
-extern void _irq119();
-extern void _irq120();
-extern void _irq121();
-extern void _irq122();
-extern void _irq123();
-extern void _irq124();
-extern void _irq125();
-extern void _irq126();
-extern void _irq127();
-extern void _irq128();
-extern void _irq129();
-extern void _irq130();
-extern void _irq131();
-extern void _irq132();
-extern void _irq133();
-extern void _irq134();
-extern void _irq135();
-extern void _irq136();
-extern void _irq137();
-extern void _irq138();
-extern void _irq139();
-extern void _irq140();
-extern void _irq141();
-extern void _irq142();
-extern void _irq143();
-extern void _irq144();
-extern void _irq145();
-extern void _irq146();
-extern void _irq147();
-extern void _irq148();
-extern void _irq149();
-extern void _irq150();
-extern void _irq151();
-extern void _irq152();
-extern void _irq153();
-extern void _irq154();
-extern void _irq155();
-extern void _irq156();
-extern void _irq157();
-extern void _irq158();
-extern void _irq159();
-extern void _irq160();
-extern void _irq161();
-extern void _irq162();
-extern void _irq163();
-extern void _irq164();
-extern void _irq165();
-extern void _irq166();
-extern void _irq167();
-extern void _irq168();
-extern void _irq169();
-extern void _irq170();
-extern void _irq171();
-extern void _irq172();
-extern void _irq173();
-extern void _irq174();
-extern void _irq175();
-extern void _irq176();
-extern void _irq177();
-extern void _irq178();
-extern void _irq179();
-extern void _irq180();
-extern void _irq181();
-extern void _irq182();
-extern void _irq183();
-extern void _irq184();
-extern void _irq185();
-extern void _irq186();
-extern void _irq187();
-extern void _irq188();
-extern void _irq189();
-extern void _irq190();
-extern void _irq191();
-extern void _irq192();
-extern void _irq193();
-extern void _irq194();
-extern void _irq195();
-extern void _irq196();
-extern void _irq197();
-extern void _irq198();
-extern void _irq199();
-extern void _irq200();
-extern void _irq201();
-extern void _irq202();
-extern void _irq203();
-extern void _irq204();
-extern void _irq205();
-extern void _irq206();
-extern void _irq207();
-extern void _irq208();
-extern void _irq209();
-extern void _irq210();
-extern void _irq211();
-extern void _irq212();
-extern void _irq213();
-extern void _irq214();
-extern void _irq215();
-extern void _irq216();
-extern void _irq217();
-extern void _irq218();
-extern void _irq219();
-extern void _irq220();
-extern void _irq221();
-extern void _irq222();
-extern void _irq223();
+extern void _irq0(void);
+extern void _irq1(void);
+extern void _irq2(void);
+extern void _irq3(void);
+extern void _irq4(void);
+extern void _irq5(void);
+extern void _irq6(void);
+extern void _irq7(void);
+extern void _irq8(void);
+extern void _irq9(void);
+extern void _irq10(void);
+extern void _irq11(void);
+extern void _irq12(void);
+extern void _irq13(void);
+extern void _irq14(void);
+extern void _irq15(void);
+extern void _irq16(void);
+extern void _irq17(void);
+extern void _irq18(void);
+extern void _irq19(void);
+extern void _irq20(void);
+extern void _irq21(void);
+extern void _irq22(void);
+extern void _irq23(void);
+extern void _irq24(void);
+extern void _irq25(void);
+extern void _irq26(void);
+extern void _irq27(void);
+extern void _irq28(void);
+extern void _irq29(void);
+extern void _irq30(void);
+extern void _irq31(void);
+extern void _irq32(void);
+extern void _irq33(void);
+extern void _irq34(void);
+extern void _irq35(void);
+extern void _irq36(void);
+extern void _irq37(void);
+extern void _irq38(void);
+extern void _irq39(void);
+extern void _irq40(void);
+extern void _irq41(void);
+extern void _irq42(void);
+extern void _irq43(void);
+extern void _irq44(void);
+extern void _irq45(void);
+extern void _irq46(void);
+extern void _irq47(void);
+extern void _irq48(void);
+extern void _irq49(void);
+extern void _irq50(void);
+extern void _irq51(void);
+extern void _irq52(void);
+extern void _irq53(void);
+extern void _irq54(void);
+extern void _irq55(void);
+extern void _irq56(void);
+extern void _irq57(void);
+extern void _irq58(void);
+extern void _irq59(void);
+extern void _irq60(void);
+extern void _irq61(void);
+extern void _irq62(void);
+extern void _irq63(void);
+extern void _irq64(void);
+extern void _irq65(void);
+extern void _irq66(void);
+extern void _irq67(void);
+extern void _irq68(void);
+extern void _irq69(void);
+extern void _irq70(void);
+extern void _irq71(void);
+extern void _irq72(void);
+extern void _irq73(void);
+extern void _irq74(void);
+extern void _irq75(void);
+extern void _irq76(void);
+extern void _irq77(void);
+extern void _irq78(void);
+extern void _irq79(void);
+extern void _irq80(void);
+extern void _irq81(void);
+extern void _irq82(void);
+extern void _irq83(void);
+extern void _irq84(void);
+extern void _irq85(void);
+extern void _irq86(void);
+extern void _irq87(void);
+extern void _irq88(void);
+extern void _irq89(void);
+extern void _irq90(void);
+extern void _irq91(void);
+extern void _irq92(void);
+extern void _irq93(void);
+extern void _irq94(void);
+extern void _irq95(void);
+extern void _irq96(void);
+extern void _irq97(void);
+extern void _irq98(void);
+extern void _irq99(void);
+extern void _irq100(void);
+extern void _irq101(void);
+extern void _irq102(void);
+extern void _irq103(void);
+extern void _irq104(void);
+extern void _irq105(void);
+extern void _irq106(void);
+extern void _irq107(void);
+extern void _irq108(void);
+extern void _irq109(void);
+extern void _irq110(void);
+extern void _irq111(void);
+extern void _irq112(void);
+extern void _irq113(void);
+extern void _irq114(void);
+extern void _irq115(void);
+extern void _irq116(void);
+extern void _irq117(void);
+extern void _irq118(void);
+extern void _irq119(void);
+extern void _irq120(void);
+extern void _irq121(void);
+extern void _irq122(void);
+extern void _irq123(void);
+extern void _irq124(void);
+extern void _irq125(void);
+extern void _irq126(void);
+extern void _irq127(void);
+extern void _irq128(void);
+extern void _irq129(void);
+extern void _irq130(void);
+extern void _irq131(void);
+extern void _irq132(void);
+extern void _irq133(void);
+extern void _irq134(void);
+extern void _irq135(void);
+extern void _irq136(void);
+extern void _irq137(void);
+extern void _irq138(void);
+extern void _irq139(void);
+extern void _irq140(void);
+extern void _irq141(void);
+extern void _irq142(void);
+extern void _irq143(void);
+extern void _irq144(void);
+extern void _irq145(void);
+extern void _irq146(void);
+extern void _irq147(void);
+extern void _irq148(void);
+extern void _irq149(void);
+extern void _irq150(void);
+extern void _irq151(void);
+extern void _irq152(void);
+extern void _irq153(void);
+extern void _irq154(void);
+extern void _irq155(void);
+extern void _irq156(void);
+extern void _irq157(void);
+extern void _irq158(void);
+extern void _irq159(void);
+extern void _irq160(void);
+extern void _irq161(void);
+extern void _irq162(void);
+extern void _irq163(void);
+extern void _irq164(void);
+extern void _irq165(void);
+extern void _irq166(void);
+extern void _irq167(void);
+extern void _irq168(void);
+extern void _irq169(void);
+extern void _irq170(void);
+extern void _irq171(void);
+extern void _irq172(void);
+extern void _irq173(void);
+extern void _irq174(void);
+extern void _irq175(void);
+extern void _irq176(void);
+extern void _irq177(void);
+extern void _irq178(void);
+extern void _irq179(void);
+extern void _irq180(void);
+extern void _irq181(void);
+extern void _irq182(void);
+extern void _irq183(void);
+extern void _irq184(void);
+extern void _irq185(void);
+extern void _irq186(void);
+extern void _irq187(void);
+extern void _irq188(void);
+extern void _irq189(void);
+extern void _irq190(void);
+extern void _irq191(void);
+extern void _irq192(void);
+extern void _irq193(void);
+extern void _irq194(void);
+extern void _irq195(void);
+extern void _irq196(void);
+extern void _irq197(void);
+extern void _irq198(void);
+extern void _irq199(void);
+extern void _irq200(void);
+extern void _irq201(void);
+extern void _irq202(void);
+extern void _irq203(void);
+extern void _irq204(void);
+extern void _irq205(void);
+extern void _irq206(void);
+extern void _irq207(void);
+extern void _irq208(void);
+extern void _irq209(void);
+extern void _irq210(void);
+extern void _irq211(void);
+extern void _irq212(void);
+extern void _irq213(void);
+extern void _irq214(void);
+extern void _irq215(void);
+extern void _irq216(void);
+extern void _irq217(void);
+extern void _irq218(void);
+extern void _irq219(void);
+extern void _irq220(void);
+extern void _irq221(void);
+extern void _irq222(void);
+extern void _irq223(void);
 
 static uintptr_t int_entry_points[NR_IDT_ENTRIES] = {
 	[0] = (uintptr_t)_isr0,     [1] = (uintptr_t)_isr1,

arch/x86_64/irqvec.S

@@ -366,7 +366,7 @@ syscall_gate:
 	movq %rsp, %gs:20       # GS+20 = rsp2 in the current TSS block (user stack storage)
 	movq %gs:4, %rsp	# GS+4  = rsp0 in the current TSS block (per-thread kstack)
 
-	# start building a pf_cpu_context
+	# start building a ml_cpu_context
 	pushq $0x1b
 	pushq %gs:20
 	push %r11

arch/x86_64/panic.c

@@ -1,8 +1,9 @@
-#include "mango/machine/panic.h"
-#include "mango/vm.h"
-#include <mango/printk.h>
-#include <mango/libc/stdio.h>
 #include <arch/irq.h>
+#include <kernel/libc/stdio.h>
+#include <kernel/machine/cpu.h>
+#include <kernel/machine/panic.h>
+#include <kernel/printk.h>
+#include <kernel/vm.h>
 
 #define R_CF  0
 #define R_PF  2
@@ -82,7 +83,11 @@ static void print_rflags(uintptr_t rflags)
 		if (rflags & (1 << i)) {
 			const char *name = pf_rfl_name(i);
 			if (name) {
-				buf_i += snprintf(buf + buf_i, sizeof(buf) - buf_i, " %s", name);
+				buf_i += snprintf(
+					buf + buf_i,
+					sizeof(buf) - buf_i,
+					" %s",
+					name);
 			}
 		}
 	}
@@ -91,30 +96,43 @@ static void print_rflags(uintptr_t rflags)
 	printk(buf);
 }
 
-void ml_print_cpu_state(struct cpu_context *ctx)
+void ml_print_cpu_state(struct ml_cpu_context *ctx)
 {
 	printk("cpu state:");
 	if (ctx) {
 		printk("  rax %016llx rbx %016llx rcx %016llx",
-		       ctx->rax, ctx->rbx, ctx->rcx);
+		       ctx->rax,
+		       ctx->rbx,
+		       ctx->rcx);
 		printk("  rdx %016llx rsi %016llx rdi %016llx",
-		       ctx->rdx, ctx->rsi, ctx->rdi);
+		       ctx->rdx,
+		       ctx->rsi,
+		       ctx->rdi);
 		printk("  rsp %016llx rbp %016llx r8  %016llx",
-		       ctx->rsp, ctx->rbp, ctx->r8);
+		       ctx->rsp,
+		       ctx->rbp,
+		       ctx->r8);
 		printk("  r9  %016llx r10 %016llx r11 %016llx",
-		       ctx->r9, ctx->r10, ctx->r11);
+		       ctx->r9,
+		       ctx->r10,
+		       ctx->r11);
 		printk("  r12 %016llx r13 %016llx r14 %016llx",
-		       ctx->r12, ctx->r13, ctx->r14);
+		       ctx->r12,
+		       ctx->r13,
+		       ctx->r14);
 		printk("  r15 %016llx rip %016llx cs  %04x ss %04x",
-		       ctx->r15, ctx->rip, ctx->cs, ctx->ss);
+		       ctx->r15,
+		       ctx->rip,
+		       ctx->cs,
+		       ctx->ss);
 		print_rflags(ctx->rflags);
 	}
 
 	uintptr_t cr0 = 0, cr2 = 0, cr3 = 0, cr4 = 0;
-	asm volatile("mov %%cr0, %%rax" : "=a" (cr0));
-	asm volatile("mov %%cr2, %%rax" : "=a" (cr2));
-	asm volatile("mov %%cr3, %%rax" : "=a" (cr3));
-	asm volatile("mov %%cr4, %%rax" : "=a" (cr4));
+	asm volatile("mov %%cr0, %%rax" : "=a"(cr0));
+	asm volatile("mov %%cr2, %%rax" : "=a"(cr2));
+	asm volatile("mov %%cr3, %%rax" : "=a"(cr3));
+	asm volatile("mov %%cr4, %%rax" : "=a"(cr4));
 	printk("  cr0 %016llx cr2 %016llx", cr0, cr2);
 	printk("  cr3 %016llx cr4 %016llx", cr3, cr4);
 }
@@ -135,7 +153,12 @@ static void print_stack_item(uintptr_t addr)
 	int found = -1;
 
 	if (found == 0 && name[0] != '\0') {
-		i += snprintf(buf + i, sizeof(buf) - i, "%s+0x%lx", name, offset);
+		i += snprintf(
+			buf + i,
+			sizeof(buf) - i,
+			"%s+0x%lx",
+			name,
+			offset);
 	} else {
 		i += snprintf(buf + i, sizeof(buf) - i, "?");
 	}
@@ -152,9 +175,8 @@ static void print_stack_trace(uintptr_t ip, uintptr_t *bp)
 
 	int max_frames = 10, current_frame = 0;
 	while (1) {
-		if (!vm_virt_to_phys(stk) ||
-		    bp == NULL ||
-		    current_frame > max_frames) {
+		if (!vm_virt_to_phys(stk) || bp == NULL
+		    || current_frame > max_frames) {
 			break;
 		}
 
@@ -169,11 +191,11 @@ static void print_stack_trace(uintptr_t ip, uintptr_t *bp)
 void ml_print_stack_trace(uintptr_t ip)
 {
 	uintptr_t *bp;
-	asm volatile("mov %%rbp, %0" : "=r" (bp));
+	asm volatile("mov %%rbp, %0" : "=r"(bp));
 	print_stack_trace(ip, bp);
 }
 
-void ml_print_stack_trace_irq(struct cpu_context *ctx)
+void ml_print_stack_trace_irq(struct ml_cpu_context *ctx)
 {
 	print_stack_trace(ctx->rip, (uintptr_t *)ctx->rbp);
 }

arch/x86_64/pit.c

@@ -1,8 +1,8 @@
 #include <arch/irq.h>
 #include <arch/ports.h>
-#include <mango/clock.h>
-#include <mango/cpu.h>
-#include <mango/printk.h>
+#include <kernel/clock.h>
+#include <kernel/cpu.h>
+#include <kernel/printk.h>
 
 #define PIT_COUNTER0  0x40
 #define PIT_CMD       0x43

arch/x86_64/pmap.c

@@ -1,17 +1,21 @@
-#include <mango/compiler.h>
-#include <mango/memblock.h>
-#include <mango/pmap.h>
-#include <mango/printk.h>
+#include <kernel/compiler.h>
+#include <kernel/libc/stdio.h>
+#include <kernel/memblock.h>
+#include <kernel/pmap.h>
+#include <kernel/printk.h>
+#include <kernel/sched.h>
+#include <kernel/types.h>
+#include <kernel/vm-object.h>
+#include <kernel/vm-region.h>
+#include <kernel/vm.h>
 #include <mango/status.h>
-#include <mango/types.h>
-#include <mango/vm.h>
 
 /* some helpful datasize constants */
 #define C_1GiB          0x40000000ULL
 #define C_2GiB          (2 * C_1GiB)
 
 #define BAD_INDEX       ((unsigned int)-1)
-#define PTR_TO_ENTRY(x) (((x) & ~VM_PAGE_MASK) | PTE_PRESENT | PTE_RW)
+#define PTR_TO_ENTRY(x) (((x) & ~VM_PAGE_MASK) | PTE_PRESENT | PTE_RW | PTE_USR)
 #define ENTRY_TO_PTR(x) ((x) & ~VM_PAGE_MASK)
 
 #define PFN(x)          ((x) >> VM_PAGE_SHIFT)
@@ -39,7 +43,7 @@ static pmap_t alloc_pmap(void)
 	return vm_virt_to_phys(p);
 }
 
-static pte_t make_pte(pfn_t pfn, enum vm_prot prot, enum page_size size)
+static pte_t make_pte(pfn_t pfn, vm_prot_t prot, enum page_size size)
 {
 	pte_t v = pfn;
 
@@ -133,12 +137,11 @@ static void delete_pdir(phys_addr_t pd)
 
 static kern_status_t do_pmap_add(
 	pmap_t pmap,
-	void *p,
+	virt_addr_t pv,
 	pfn_t pfn,
-	enum vm_prot prot,
+	vm_prot_t prot,
 	enum page_size size)
 {
-	uintptr_t pv = (uintptr_t)p;
 	unsigned int pml4t_index = BAD_INDEX, pdpt_index = BAD_INDEX,
 		     pd_index = BAD_INDEX, pt_index = BAD_INDEX;
 
@@ -261,11 +264,11 @@ void pmap_bootstrap(void)
 	/* map 2GiB at the end of the address space to
 	   replace the mapping created by start_32 and allow access to
 	   the kernel and memblock-allocated data. */
-	uintptr_t vbase = VM_KERNEL_VOFFSET;
+	virt_addr_t vbase = VM_KERNEL_VOFFSET;
 	for (size_t i = 0; i < C_2GiB; i += hugepage_sz) {
 		do_pmap_add(
 			kernel_pmap,
-			(void *)(vbase + i),
+			vbase + i,
 			PFN(i),
 			VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXEC
 				| VM_PROT_SVR,
@@ -285,7 +288,7 @@ void pmap_bootstrap(void)
 	for (size_t i = 0; i < pmem_limit; i += hugepage_sz) {
 		do_pmap_add(
 			kernel_pmap,
-			(void *)(vbase + i),
+			vbase + i,
 			PFN(i),
 			VM_PROT_READ | VM_PROT_WRITE | VM_PROT_SVR
 				| VM_PROT_NOCACHE,
@@ -297,18 +300,78 @@ void pmap_bootstrap(void)
 
 pmap_t pmap_create(void)
 {
-	return 0;
+	pmap_t pmap = alloc_pmap();
+	pmap_t kernel_pmap = get_kernel_pmap();
+
+	struct pml4t *pml4t = vm_phys_to_virt(pmap);
+	struct pml4t *kernel_pml4t = vm_phys_to_virt(kernel_pmap);
+
+	for (unsigned int i = 256; i < 512; i++) {
+		pml4t->p_entries[i] = kernel_pml4t->p_entries[i];
+	}
+
+	return pmap;
 }
 
 void pmap_destroy(pmap_t pmap)
 {
 }
 
+static void log_fault(virt_addr_t fault_addr, enum pmap_fault_flags flags)
+{
+	char flag_str[128] = {0};
+	size_t p = 0;
+
+	if (flags & PMAP_FAULT_PRESENT) {
+		p += snprintf(flag_str + p, sizeof flag_str - p, " PRESENT");
+	} else {
+		p += snprintf(flag_str + p, sizeof flag_str - p, " MISSING");
+	}
+
+	if (flags & PMAP_FAULT_USER) {
+		p += snprintf(flag_str + p, sizeof flag_str - p, " USER");
+	} else {
+		p += snprintf(flag_str + p, sizeof flag_str - p, " SVR");
+	}
+
+	if (flags & PMAP_FAULT_WRITE) {
+		p += snprintf(flag_str + p, sizeof flag_str - p, " WRITE");
+	} else {
+		p += snprintf(flag_str + p, sizeof flag_str - p, " READ");
+	}
+
+	if (flags & PMAP_FAULT_IFETCH) {
+		p += snprintf(flag_str + p, sizeof flag_str - p, " IFETCH");
+	}
+
+	if (flags & PMAP_FAULT_BADCFG) {
+		p += snprintf(flag_str + p, sizeof flag_str - p, " BADCFG");
+	}
+
+	printk("pmap: fault at 0x%llx (%s)", fault_addr, flag_str);
+}
+
+kern_status_t pmap_handle_fault(
+	virt_addr_t fault_addr,
+	enum pmap_fault_flags flags)
+{
+	// log_fault(fault_addr, flags);
+
+	if (flags & PMAP_FAULT_PRESENT) {
+		return KERN_FATAL_ERROR;
+	}
+
+	struct task *task = current_task();
+	struct vm_region *space = task->t_address_space;
+
+	return vm_region_demand_map(space, fault_addr, flags);
+}
+
 kern_status_t pmap_add(
 	pmap_t pmap,
-	void *p,
+	virt_addr_t p,
 	pfn_t pfn,
-	enum vm_prot prot,
+	vm_prot_t prot,
 	enum pmap_flags flags)
 {
 	enum page_size ps = PS_4K;
@@ -321,21 +384,21 @@ kern_status_t pmap_add(
 
 kern_status_t pmap_add_block(
 	pmap_t pmap,
-	void *p,
+	virt_addr_t p,
 	pfn_t pfn,
 	size_t len,
-	enum vm_prot prot,
+	vm_prot_t prot,
 	enum pmap_flags flags)
 {
 	return KERN_OK;
 }
 
-kern_status_t pmap_remove(pmap_t pmap, void *p)
+kern_status_t pmap_remove(pmap_t pmap, virt_addr_t p)
 {
 	return KERN_OK;
 }
 
-kern_status_t pmap_remove_range(pmap_t pmap, void *p, size_t len)
+kern_status_t pmap_remove_range(pmap_t pmap, virt_addr_t p, size_t len)
 {
 	return KERN_OK;
 }

arch/x86_64/serial.c

@@ -1,8 +1,8 @@
 #include <arch/irq.h>
 #include <arch/ports.h>
 #include <arch/serial.h>
-#include <mango/libc/stdio.h>
-#include <mango/printk.h>
+#include <kernel/libc/stdio.h>
+#include <kernel/printk.h>
 
 #define COM1 0x3F8
 #define COM2 0x2F8

arch/x86_64/thread.c

@@ -1,12 +1,24 @@
-#include <mango/machine/thread.h>
+#include <kernel/machine/cpu.h>
+#include <kernel/machine/thread.h>
 
+#define MAX_REG_ARGS 6
+#define REG_ARG_0    rdi
+#define REG_ARG_1    rsi
+#define REG_ARG_2    rdx
+#define REG_ARG_3    rcx
+#define REG_ARG_4    r8
+#define REG_ARG_5    r9
+
+/* this is the context information restored by ml_thread_switch.
+ * since ml_thread_switch only jumps to kernel-mode, IRETQ isn't used,
+ * and the extra register values needed by IRETQ aren't present. */
 struct thread_ctx {
 	uint64_t r15, r14, r13, r12, r11, r10, r9, r8;
 	uint64_t rdi, rsi, rbp, unused_rsp, rbx, rdx, rcx, rax;
 	uint64_t rfl;
 } __packed;
 
-void prepare_stack(uintptr_t ip, uintptr_t *sp)
+void ml_thread_prepare_kernel_context(uintptr_t ip, uintptr_t *sp)
 {
 	(*sp) -= sizeof(uintptr_t);
 	uintptr_t *dest_ip = (uintptr_t *)(*sp);
@@ -18,3 +30,50 @@ void prepare_stack(uintptr_t ip, uintptr_t *sp)
 
 	ctx->rfl = 0x202;
 }
+
+extern kern_status_t ml_thread_prepare_user_context(
+	virt_addr_t ip,
+	virt_addr_t user_sp,
+	virt_addr_t *kernel_sp,
+	const uintptr_t *args,
+	size_t nr_args)
+{
+	(*kernel_sp) -= sizeof(struct ml_cpu_context);
+
+	struct ml_cpu_context *ctx = (struct ml_cpu_context *)(*kernel_sp);
+	memset(ctx, 0x0, sizeof *ctx);
+	ctx->rip = ip;
+	ctx->rsp = user_sp;
+	ctx->ss = 0x1b;
+	ctx->cs = 0x23;
+	ctx->rflags = 0x202;
+	ctx->rdi = 0;   // arg 0
+	ctx->rsi = 0;   // arg 1
+
+	for (size_t i = 0; i < nr_args; i++) {
+		switch (i) {
+		case 0:
+			ctx->REG_ARG_0 = args[i];
+			break;
+		case 1:
+			ctx->REG_ARG_1 = args[i];
+			break;
+		case 2:
+			ctx->REG_ARG_2 = args[i];
+			break;
+		case 3:
+			ctx->REG_ARG_3 = args[i];
+			break;
+		case 4:
+			ctx->REG_ARG_4 = args[i];
+			break;
+		case 5:
+			ctx->REG_ARG_5 = args[i];
+			break;
+		default:
+			return KERN_INVALID_ARGUMENT;
+		}
+	}
+
+	return KERN_OK;
+}

arch/x86_64/thread_switch.S

@@ -1,13 +1,13 @@
-	.code64
+.code64
 
-	.extern THREAD_sp
+.extern THREAD_sp
 
-	.global switch_to
-	.type switch_to, @function
+.global ml_thread_switch
+.type ml_thread_switch, @function
 
 	// %rdi = (struct thread *) current thread.
 	// %rsi = (struct thread *) next thread.
-switch_to:
+ml_thread_switch:
 	pushfq
 
 	push %rax
@@ -50,3 +50,27 @@ switch_to:
 	popfq
 
 	ret
+
+
+.global ml_thread_switch_user
+.type ml_thread_switch_user, @function
+ml_thread_switch_user:
+	pop %r15
+	pop %r14
+	pop %r13
+	pop %r12
+	pop %r11
+	pop %r10
+	pop %r9
+	pop %r8
+	pop %rdi
+	pop %rsi
+	pop %rbp
+	add $8, %rsp
+	pop %rbx
+	pop %rdx
+	pop %rcx
+	pop %rax
+
+	add $16, %rsp
+	iretq

arch/x86_64/tss.c

@@ -0,0 +1,48 @@
+#include "arch/msr.h"
+
+#include <arch/gdt.h>
+#include <arch/tss.h>
+#include <kernel/libc/string.h>
+
+static void tss_flush(int index)
+{
+	index *= sizeof(struct gdt_entry);
+	index |= 3;
+	asm volatile("mov %0, %%eax; ltr %%ax" ::"r"(index));
+}
+
+void tss_init(struct tss *tss, struct tss_ptr *ptr)
+{
+	memset(tss, 0x0, sizeof *tss);
+
+	ptr->tss_base = (uint64_t)tss;
+	ptr->tss_limit = (uint16_t)sizeof *tss;
+}
+
+void tss_load(struct tss *tss)
+{
+	tss_flush(TSS_GDT_INDEX);
+
+	uintptr_t kernel_gs_base_reg = 0xC0000102;
+	wrmsr(kernel_gs_base_reg, (uintptr_t)tss);
+}
+
+virt_addr_t tss_get_kstack(struct tss *tss)
+{
+	return tss->rsp0;
+}
+
+virt_addr_t tss_get_ustack(struct tss *tss)
+{
+	return tss->rsp2;
+}
+
+void tss_set_kstack(struct tss *tss, virt_addr_t sp)
+{
+	tss->rsp0 = sp;
+}
+
+void tss_set_ustack(struct tss *tss, virt_addr_t sp)
+{
+	tss->rsp2 = sp;
+}

arch/x86_64/vga.c

@@ -1,9 +1,9 @@
 #include <arch/irq.h>
 #include <arch/ports.h>
 #include <arch/serial.h>
-#include <mango/libc/stdio.h>
-#include <mango/machine/vm.h>
-#include <mango/printk.h>
+#include <kernel/libc/stdio.h>
+#include <kernel/machine/vm.h>
+#include <kernel/printk.h>
 
 struct vga_console {
 	uint16_t *vga_framebuffer;

ds/bitmap.c

@@ -1,5 +1,5 @@
-#include <mango/libc/string.h>
-#include <mango/bitmap.h>
+#include <kernel/bitmap.h>
+#include <kernel/libc/string.h>
 
 void bitmap_zero(unsigned long *map, unsigned long nbits)
 {
@@ -25,7 +25,7 @@ void bitmap_set(unsigned long *map, unsigned long bit)
 void bitmap_clear(unsigned long *map, unsigned long bit)
 {
 	unsigned long index = bit / BITS_PER_WORD;
-	unsigned long offset = bit & (BITS_PER_WORD - 1);
+	unsigned long offset = (BITS_PER_WORD - bit - 1) & (BITS_PER_WORD - 1);
 	unsigned long mask = 1ul << offset;
 
 	map[index] &= ~mask;
@@ -38,7 +38,6 @@ bool bitmap_check(unsigned long *map, unsigned long bit)
 	unsigned long mask = 1ul << offset;
 
 	return (map[index] & mask) != 0 ? true : false;
-
 }
 
 unsigned int bitmap_count_set(unsigned long *map, unsigned long nbits)

ds/btree.c

@@ -57,7 +57,7 @@
    provide a comparator function.
 */ 
    
-#include <mango/btree.h>
+#include <kernel/btree.h>
 #include <stddef.h>
 
 #define MAX(a, b) ((a) > (b) ? (a) : (b))

ds/queue.c

@@ -1,4 +1,4 @@
-#include <mango/queue.h>
+#include <kernel/queue.h>
 
 size_t queue_length(struct queue *q)
 {

ds/ringbuffer.c

@@ -1,5 +1,5 @@
-#include <mango/ringbuffer.h>
-#include <mango/sched.h>
+#include <kernel/ringbuffer.h>
+#include <kernel/sched.h>
 
 size_t ringbuffer_unread(struct ringbuffer *ring_buffer)
 {

include/kernel/arg.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_ARG_H_
-#define MANGO_ARG_H_
+#ifndef KERNEL_ARG_H_
+#define KERNEL_ARG_H_
 
 #include <stdbool.h>
 #include <mango/status.h>

include/kernel/bitmap.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_BITMAP_H_
-#define MANGO_BITMAP_H_
+#ifndef KERNEL_BITMAP_H_
+#define KERNEL_BITMAP_H_
 
 #include <stdbool.h>
 

include/kernel/bsp.h

@@ -0,0 +1,38 @@
+#ifndef KERNEL_BSP_H_
+#define KERNEL_BSP_H_
+
+#include <kernel/compiler.h>
+#include <mango/status.h>
+#include <kernel/types.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#define BSP_MAGIC 0xcafebabe
+
+struct task;
+
+struct bsp_trailer {
+	/* these fields are stored in big endian in the package itself */
+	uint32_t bsp_magic;
+	uint64_t bsp_fs_offset;
+	uint32_t bsp_fs_len;
+	uint64_t bsp_exec_offset;
+	uint32_t bsp_exec_len;
+	uint64_t bsp_text_faddr, bsp_text_vaddr, bsp_text_size;
+	uint64_t bsp_data_faddr, bsp_data_vaddr, bsp_data_size;
+	uint64_t bsp_exec_entry;
+} __packed;
+
+struct bsp {
+	/* the values in this struct are stored in host byte order */
+	struct bsp_trailer bsp_trailer;
+	struct vm_object *bsp_vmo;
+};
+
+extern void bsp_set_location(const struct boot_module *mod);
+extern void bsp_get_location(struct boot_module *out);
+
+extern kern_status_t bsp_load(struct bsp *bsp, const struct boot_module *mod);
+extern kern_status_t bsp_launch_async(struct bsp *bsp, struct task *task);
+
+#endif

include/kernel/btree.h

@@ -20,26 +20,34 @@
     software without specific prior written permission.
  */
 
-#ifndef MANGO_BTREE_H_
-#define MANGO_BTREE_H_
+#ifndef KERNEL_BTREE_H_
+#define KERNEL_BTREE_H_
 
+#include <stdbool.h>
+#include <stddef.h>
 #include <stdint.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-/* if your custom structure contains a struct btree_node (i.e. it can be part of a btree),
-   you can use this macro to convert a struct btree_node* to a your_type*
+/* if your custom structure contains a struct btree_node (i.e. it can be part of
+   a btree), you can use this macro to convert a struct btree_node* to a
+   your_type*
 
-   @param t the name of your custom type (something that can be passed to offsetof)
-   @param m the name of the struct btree_node member variable within your custom type.
-   @param v the struct btree_node pointer that you wish to convert. if this is NULL, NULL will be returned.
+   @param t the name of your custom type (something that can be passed to
+   offsetof)
+   @param m the name of the struct btree_node member variable within your custom
+   type.
+   @param v the struct btree_node pointer that you wish to convert. if this is
+   NULL, NULL will be returned.
 */
-#define BTREE_CONTAINER(t, m, v) ((void *)((v) ? (uintptr_t)(v) - (offsetof(t, m)) : 0))
+#define BTREE_CONTAINER(t, m, v)                                               \
+	((void *)((v) ? (uintptr_t)(v) - (offsetof(t, m)) : 0))
 
 /* defines a simple node insertion function.
-   this function assumes that your nodes have simple integer keys that can be compared with the usual operators.
+   this function assumes that your nodes have simple integer keys that can be
+   compared with the usual operators.
 
    EXAMPLE:
    if you have a tree node type like this:
@@ -49,45 +57,67 @@ extern "C" {
              struct btree_node base;
      }
 
-   You would use the following call to generate an insert function for a tree with this node type:
+   You would use the following call to generate an insert function for a tree
+   with this node type:
 
-     BTREE_DEFINE_SIMPLE_INSERT(struct my_tree_node, base, key, my_tree_node_insert);
+     BTREE_DEFINE_SIMPLE_INSERT(struct my_tree_node, base, key,
+   my_tree_node_insert);
 
    Which would emit a function defined like:
 
-     static void my_tree_node_insert(struct btree *tree, struct my_tree_node *node);
+     static void my_tree_node_insert(struct btree *tree, struct my_tree_node
+   *node);
 
-   @param node_type your custom tree node type. usually a structure that contains a struct btree_node member.
-   @param container_node_member the name of the struct btree_node member variable within your custom type.
-   @param container_key_member the name of the key member variable within your custom type.
+   @param node_type your custom tree node type. usually a structure that
+   contains a struct btree_node member.
+   @param container_node_member the name of the struct btree_node member
+   variable within your custom type.
+   @param container_key_member the name of the key member variable within your
+   custom type.
    @param function_name the name of the function to generate.
 */
-#define BTREE_DEFINE_SIMPLE_INSERT(node_type, container_node_member, container_key_member, function_name) \
+#define BTREE_DEFINE_SIMPLE_INSERT(                                            \
+	node_type,                                                             \
+	container_node_member,                                                 \
+	container_key_member,                                                  \
+	function_name)                                                         \
 	void function_name(struct btree *tree, node_type *node)                \
 	{                                                                      \
 		if (!tree->b_root) {                                           \
 			tree->b_root = &node->container_node_member;           \
-			btree_insert_fixup(tree, &node->container_node_member);                           \
+			btree_insert_fixup(                                    \
+				tree,                                          \
+				&node->container_node_member);                 \
 			return;                                                \
 		}                                                              \
                                                                                \
 		struct btree_node *cur = tree->b_root;                         \
 		while (1) {                                                    \
-			node_type *cur_node = BTREE_CONTAINER(node_type, container_node_member, cur);     \
+			node_type *cur_node = BTREE_CONTAINER(                 \
+				node_type,                                     \
+				container_node_member,                         \
+				cur);                                          \
 			struct btree_node *next = NULL;                        \
                                                                                \
-			if (node->container_key_member > cur_node->container_key_member) {                \
+			if (node->container_key_member                         \
+			    > cur_node->container_key_member) {                \
 				next = btree_right(cur);                       \
                                                                                \
 				if (!next) {                                   \
-					btree_put_right(cur, &node->container_node_member);               \
+					btree_put_right(                       \
+						cur,                           \
+						&node->container_node_member); \
 					break;                                 \
 				}                                              \
-			} else if (node->container_key_member < cur_node->container_key_member) {         \
+			} else if (                                            \
+				node->container_key_member                     \
+				< cur_node->container_key_member) {            \
 				next = btree_left(cur);                        \
                                                                                \
 				if (!next) {                                   \
-					btree_put_left(cur, &node->container_node_member);                \
+					btree_put_left(                        \
+						cur,                           \
+						&node->container_node_member); \
 					break;                                 \
 				}                                              \
 			} else {                                               \
@@ -101,8 +131,8 @@ extern "C" {
 	}
 
 /* defines a node insertion function.
-   this function should be used for trees with complex node keys that cannot be directly compared.
-   a comparator for your keys must be supplied.
+   this function should be used for trees with complex node keys that cannot be
+   directly compared. a comparator for your keys must be supplied.
 
    EXAMPLE:
    if you have a tree node type like this:
@@ -112,7 +142,8 @@ extern "C" {
              struct btree_node base;
      }
 
-   You would need to define a comparator function or macro with the following signature:
+   You would need to define a comparator function or macro with the following
+   signature:
 
      int my_comparator(struct my_tree_node *a, struct my_tree_node *b);
 
@@ -122,33 +153,49 @@ extern "C" {
      return  0   if a == b
      return  1   if a > b
 
-   You would use the following call to generate an insert function for a tree with this node type:
+   You would use the following call to generate an insert function for a tree
+   with this node type:
 
-     BTREE_DEFINE_INSERT(struct my_tree_node, base, key, my_tree_node_insert, my_comparator);
+     BTREE_DEFINE_INSERT(struct my_tree_node, base, key, my_tree_node_insert,
+   my_comparator);
 
    Which would emit a function defined like:
 
-     static void my_tree_node_insert(struct btree *tree, struct my_tree_node *node);
+     static void my_tree_node_insert(struct btree *tree, struct my_tree_node
+   *node);
 
-   @param node_type your custom tree node type. usually a structure that contains a struct btree_node member.
-   @param container_node_member the name of the struct btree_node member variable within your custom type.
-   @param container_key_member the name of the key member variable within your custom type.
+   @param node_type your custom tree node type. usually a structure that
+   contains a struct btree_node member.
+   @param container_node_member the name of the struct btree_node member
+   variable within your custom type.
+   @param container_key_member the name of the key member variable within your
+   custom type.
    @param function_name the name of the function to generate.
-   @param comparator the name of a comparator function or functional-macro that conforms to the
-                     requirements listed above.
+   @param comparator the name of a comparator function or functional-macro that
+   conforms to the requirements listed above.
 */
-#define BTREE_DEFINE_INSERT(node_type, container_node_member, container_key_member, function_name, comparator) \
+#define BTREE_DEFINE_INSERT(                                                   \
+	node_type,                                                             \
+	container_node_member,                                                 \
+	container_key_member,                                                  \
+	function_name,                                                         \
+	comparator)                                                            \
 	void function_name(struct btree *tree, node_type *node)                \
 	{                                                                      \
 		if (!tree->b_root) {                                           \
 			tree->b_root = &node->container_node_member;           \
-			btree_insert_fixup(tree, &node->container_node_member);                                \
+			btree_insert_fixup(                                    \
+				tree,                                          \
+				&node->container_node_member);                 \
 			return;                                                \
 		}                                                              \
                                                                                \
 		struct btree_node *cur = tree->b_root;                         \
 		while (1) {                                                    \
-			node_type *cur_node = BTREE_CONTAINER(node_type, container_node_member, cur);          \
+			node_type *cur_node = BTREE_CONTAINER(                 \
+				node_type,                                     \
+				container_node_member,                         \
+				cur);                                          \
 			struct btree_node *next = NULL;                        \
 			int cmp = comparator(node, cur_node);                  \
                                                                                \
@@ -156,14 +203,18 @@ extern "C" {
 				next = btree_right(cur);                       \
                                                                                \
 				if (!next) {                                   \
-					btree_put_right(cur, &node->container_node_member);                    \
+					btree_put_right(                       \
+						cur,                           \
+						&node->container_node_member); \
 					break;                                 \
 				}                                              \
 			} else if (cmp == -1) {                                \
 				next = btree_left(cur);                        \
                                                                                \
 				if (!next) {                                   \
-					btree_put_left(cur, &node->container_node_member);                     \
+					btree_put_left(                        \
+						cur,                           \
+						&node->container_node_member); \
 					break;                                 \
 				}                                              \
 			} else {                                               \
@@ -177,7 +228,8 @@ extern "C" {
 	}
 
 /* defines a simple tree search function.
-   this function assumes that your nodes have simple integer keys that can be compared with the usual operators.
+   this function assumes that your nodes have simple integer keys that can be
+   compared with the usual operators.
 
    EXAMPLE:
    if you have a tree node type like this:
@@ -187,27 +239,40 @@ extern "C" {
              struct btree_node base;
      }
 
-   You would use the following call to generate a search function for a tree with this node type:
+   You would use the following call to generate a search function for a tree
+   with this node type:
 
-     BTREE_DEFINE_SIMPLE_GET(struct my_tree_node, int, base, key, my_tree_node_get);
+     BTREE_DEFINE_SIMPLE_GET(struct my_tree_node, int, base, key,
+   my_tree_node_get);
 
    Which would emit a function defined like:
 
      static struct my_tree_node *my_tree_node_get(struct btree *tree, int key);
 
-   @param node_type your custom tree node type. usually a structure that contains a struct btree_node member.
-   @param key_type the type name of the key embedded in your custom tree node type. this type must be
-                   compatible with the builtin comparison operators.
-   @param container_node_member the name of the struct btree_node member variable within your custom type.
-   @param container_key_member the name of the key member variable within your custom type.
+   @param node_type your custom tree node type. usually a structure that
+   contains a struct btree_node member.
+   @param key_type the type name of the key embedded in your custom tree node
+   type. this type must be compatible with the builtin comparison operators.
+   @param container_node_member the name of the struct btree_node member
+   variable within your custom type.
+   @param container_key_member the name of the key member variable within your
+   custom type.
    @param function_name the name of the function to generate.
 */
-#define BTREE_DEFINE_SIMPLE_GET(node_type, key_type, container_node_member, container_key_member, function_name) \
+#define BTREE_DEFINE_SIMPLE_GET(                                               \
+	node_type,                                                             \
+	key_type,                                                              \
+	container_node_member,                                                 \
+	container_key_member,                                                  \
+	function_name)                                                         \
 	node_type *function_name(struct btree *tree, key_type key)             \
 	{                                                                      \
 		struct btree_node *cur = tree->b_root;                         \
 		while (cur) {                                                  \
-			node_type *cur_node = BTREE_CONTAINER(node_type, container_node_member, cur);            \
+			node_type *cur_node = BTREE_CONTAINER(                 \
+				node_type,                                     \
+				container_node_member,                         \
+				cur);                                          \
 			if (key > cur_node->container_key_member) {            \
 				cur = btree_right(cur);                        \
 			} else if (key < cur_node->container_key_member) {     \
@@ -241,15 +306,23 @@ extern "C" {
 
      btree_foreach (struct my_tree_node, node, &my_tree, base) { ... }
 
-   @param iter_type the type name of the iterator variable. this should be the tree's node type, and shouldn't be a pointer.
+   @param iter_type the type name of the iterator variable. this should be the
+   tree's node type, and shouldn't be a pointer.
    @param iter_name the name of the iterator variable.
    @param tree_name a pointer to the tree to traverse.
-   @param node_member the name of the struct btree_node member variable within the tree node type.
+   @param node_member the name of the struct btree_node member variable within
+   the tree node type.
 */
 #define btree_foreach(iter_type, iter_name, tree_name, node_member)            \
-	for (iter_type *iter_name = BTREE_CONTAINER(iter_type, node_member, btree_first(tree_name)); \
+	for (iter_type *iter_name = BTREE_CONTAINER(                           \
+		     iter_type,                                                \
+		     node_member,                                              \
+		     btree_first(tree_name));                                  \
 	     iter_name;                                                        \
-	     iter_name = BTREE_CONTAINER(iter_type, node_member, btree_next(&((iter_name)->node_member))))
+	     iter_name = BTREE_CONTAINER(                                      \
+		     iter_type,                                                \
+		     node_member,                                              \
+		     btree_next(&((iter_name)->node_member))))
 
 /* perform an reverse in-order traversal of a binary tree
 
@@ -272,35 +345,43 @@ extern "C" {
 
      btree_foreach_r (struct my_tree_node, node, &my_tree, base) { ... }
 
-   @param iter_type the type name of the iterator variable. this should be the tree's node type, and shouldn't be a pointer.
+   @param iter_type the type name of the iterator variable. this should be the
+   tree's node type, and shouldn't be a pointer.
    @param iter_name the name of the iterator variable.
    @param tree_name a pointer to the tree to traverse.
-   @param node_member the name of the struct btree_node member variable within the tree node type.
+   @param node_member the name of the struct btree_node member variable within
+   the tree node type.
 */
 #define btree_foreach_r(iter_type, iter_name, tree_name, node_member)          \
-	for (iter_type *iter_name = BTREE_CONTAINER(iter_type, node_member, btree_last(tree_name)); \
+	for (iter_type *iter_name                                              \
+	     = BTREE_CONTAINER(iter_type, node_member, btree_last(tree_name)); \
 	     iter_name;                                                        \
-	     iter_name = BTREE_CONTAINER(iter_type, node_member, btree_prev(&((iter_name)->node_member))))
+	     iter_name = BTREE_CONTAINER(                                      \
+		     iter_type,                                                \
+		     node_member,                                              \
+		     btree_prev(&((iter_name)->node_member))))
 
-/* binary tree nodes. this *cannot* be used directly. you need to define a custom node type
-   that contains a member variable of type struct btree_node.
+/* binary tree nodes. this *cannot* be used directly. you need to define a
+   custom node type that contains a member variable of type struct btree_node.
 
-   you would then use the supplied macros to define functions to manipulate your custom binary tree.
+   you would then use the supplied macros to define functions to manipulate your
+   custom binary tree.
 */
 struct btree_node {
 	struct btree_node *b_parent, *b_left, *b_right;
 	unsigned short b_height;
 };
 
-/* binary tree. unlike struct btree_node, you can define variables of type struct btree. */
+/* binary tree. unlike struct btree_node, you can define variables of type
+ * struct btree. */
 struct btree {
 	struct btree_node *b_root;
 };
 
 /* re-balance a binary tree after an insertion operation.
 
-   NOTE that, if you define an insertion function using BTREE_DEFINE_INSERT or similar,
-   this function will automatically called for you.
+   NOTE that, if you define an insertion function using BTREE_DEFINE_INSERT or
+   similar, this function will automatically called for you.
 
    @param tree the tree to re-balance.
    @param node the node that was just inserted into the tree.
@@ -316,29 +397,42 @@ extern void btree_delete(struct btree *tree, struct btree_node *node);
 
 /* get the first node in a binary tree.
 
-   this will be the node with the smallest key (i.e. the node that is furthest-left from the root)
+   this will be the node with the smallest key (i.e. the node that is
+   furthest-left from the root)
 */
 extern struct btree_node *btree_first(struct btree *tree);
 
 /* get the last node in a binary tree.
 
-   this will be the node with the largest key (i.e. the node that is furthest-right from the root)
+   this will be the node with the largest key (i.e. the node that is
+   furthest-right from the root)
 */
 extern struct btree_node *btree_last(struct btree *tree);
-/* for any binary tree node, this function returns the node with the next-largest key value */
+/* for any binary tree node, this function returns the node with the
+ * next-largest key value */
 extern struct btree_node *btree_next(struct btree_node *node);
-/* for any binary tree node, this function returns the node with the next-smallest key value */
+/* for any binary tree node, this function returns the node with the
+ * next-smallest key value */
 extern struct btree_node *btree_prev(struct btree_node *node);
 
+static inline bool btree_empty(const struct btree *tree)
+{
+	return tree->b_root == NULL;
+}
+
 /* sets `child` as the immediate left-child of `parent` */
-static inline void btree_put_left(struct btree_node *parent, struct btree_node *child)
+static inline void btree_put_left(
+	struct btree_node *parent,
+	struct btree_node *child)
 {
 	parent->b_left = child;
 	child->b_parent = parent;
 }
 
 /* sets `child` as the immediate right-child of `parent` */
-static inline void btree_put_right(struct btree_node *parent, struct btree_node *child)
+static inline void btree_put_right(
+	struct btree_node *parent,
+	struct btree_node *child)
 {
 	parent->b_right = child;
 	child->b_parent = parent;

include/kernel/channel.h

@@ -0,0 +1,53 @@
+#ifndef KERNEL_CHANNEL_H_
+#define KERNEL_CHANNEL_H_
+
+#include <kernel/object.h>
+#include <kernel/sched.h>
+
+struct kmsg;
+
+struct channel {
+	struct object c_base;
+	unsigned int c_id;
+	struct waitqueue c_wq;
+	struct btree c_msg;
+	struct btree_node c_node;
+};
+
+extern kern_status_t channel_type_init(void);
+
+extern struct channel *channel_create(void);
+
+extern kern_status_t channel_enqueue_msg(
+	struct channel *channel,
+	struct kmsg *msg);
+
+extern kern_status_t channel_recv_msg(
+	struct channel *channel,
+	struct msg *out_msg,
+	msgid_t *out_id,
+	unsigned long *irq_flags);
+extern kern_status_t channel_reply_msg(
+	struct channel *channel,
+	msgid_t id,
+	const struct msg *resp,
+	unsigned long *irq_flags);
+
+extern kern_status_t channel_read_msg(
+	struct channel *channel,
+	msgid_t msg,
+	size_t offset,
+	void *buf,
+	size_t len,
+	size_t *nr_read);
+extern kern_status_t channel_write_msg(
+	struct channel *channel,
+	msgid_t msg,
+	size_t offset,
+	const void *buf,
+	size_t len,
+	size_t *nr_written);
+
+DEFINE_OBJECT_LOCK_FUNCTION(channel, c_base)
+
+#endif

include/kernel/clock.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_CLOCK_H_
-#define MANGO_CLOCK_H_
+#ifndef KERNEL_CLOCK_H_
+#define KERNEL_CLOCK_H_
 
 #include <stdint.h>
 

include/kernel/compiler.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_COMPILER_H_
-#define MANGO_COMPILER_H_
+#ifndef KERNEL_COMPILER_H_
+#define KERNEL_COMPILER_H_
 
 #ifdef __cplusplus
 template <typename T>

include/kernel/console.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_CONSOLE_H_
-#define MANGO_CONSOLE_H_
+#ifndef KERNEL_CONSOLE_H_
+#define KERNEL_CONSOLE_H_
 
 /* The console system
 
@@ -14,8 +14,8 @@
    representing a serial port may allow both sending AND receiving over the
    port.
 */
-#include <mango/queue.h>
-#include <mango/locks.h>
+#include <kernel/queue.h>
+#include <kernel/locks.h>
 #include <mango/status.h>
 
 #ifdef __cplusplus

include/kernel/cpu.h

@@ -1,10 +1,10 @@
-#ifndef MANGO_CPU_H_
-#define MANGO_CPU_H_
+#ifndef KERNEL_CPU_H_
+#define KERNEL_CPU_H_
 
-#include <mango/types.h>
-#include <mango/machine/cpu.h>
+#include <kernel/types.h>
+#include <kernel/machine/cpu.h>
 #include <stdint.h>
-#include <mango/sched.h>
+#include <kernel/sched.h>
 
 #ifdef __cplusplus
 extern "C" {

include/kernel/fb.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_FB_H_
-#define MANGO_FB_H_
+#ifndef KERNEL_FB_H_
+#define KERNEL_FB_H_
 
 #include <stdint.h>
 

include/kernel/flags.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_FLAGS_H_
-#define MANGO_FLAGS_H_
+#ifndef KERNEL_FLAGS_H_
+#define KERNEL_FLAGS_H_
 
 #include <stdint.h>
 

include/kernel/handle.h

@@ -0,0 +1,65 @@
+#ifndef KERNEL_HANDLE_H_
+#define KERNEL_HANDLE_H_
+
+#include <kernel/bitmap.h>
+#include <mango/status.h>
+#include <stddef.h>
+#include <stdint.h>
+
+/* subtract 32 bytes to account for the handle bitmap */
+#define HANDLES_PER_TABLE ((4096 - 32) / sizeof(struct handle))
+#define REFS_PER_TABLE    ((4096 - 64) / sizeof(struct handle_table *))
+
+typedef uint32_t kern_handle_t;
+
+typedef uintptr_t handle_flags_t;
+
+struct task;
+struct object;
+struct handle_list;
+
+struct handle {
+	struct object *h_object;
+	handle_flags_t h_flags;
+};
+
+struct handle_table {
+	union {
+		struct {
+			/* bitmap tracks which bits in t_handle_list are
+			 * allocated */
+			DECLARE_BITMAP(t_handle_map, HANDLES_PER_TABLE);
+			struct handle t_handle_list[HANDLES_PER_TABLE];
+		} t_handles;
+
+		struct {
+			/* bitmap tracks which sub-tables are fully-allocated */
+			DECLARE_BITMAP(t_subtable_map, REFS_PER_TABLE);
+			struct handle_table *t_subtable_list[REFS_PER_TABLE];
+		} t_subtables;
+	};
+};
+
+extern struct handle_table *handle_table_create(void);
+extern void handle_table_destroy(struct handle_table *tab);
+
+extern kern_status_t handle_table_alloc_handle(
+	struct handle_table *tab,
+	struct handle **out_slot,
+	kern_handle_t *out_handle);
+extern kern_status_t handle_table_free_handle(
+	struct handle_table *tab,
+	kern_handle_t handle);
+extern struct handle *handle_table_get_handle(
+	struct handle_table *tab,
+	kern_handle_t handle);
+
+extern kern_status_t handle_list_transfer(
+	struct handle_table *dest,
+	struct handle_list *dest_list,
+	size_t dest_list_count,
+	struct handle_table *src,
+	const struct handle_list *src_list,
+	size_t src_list_count);
+
+#endif

include/kernel/init.h

@@ -1,8 +1,8 @@
-#ifndef MANGO_INIT_H_
-#define MANGO_INIT_H_
+#ifndef KERNEL_INIT_H_
+#define KERNEL_INIT_H_
 
-#include <mango/compiler.h>
-#include <mango/machine/init.h>
+#include <kernel/compiler.h>
+#include <kernel/machine/init.h>
 
 #ifdef __cplusplus
 extern "C" {

include/kernel/input.h

@@ -1,8 +1,8 @@
-#ifndef MANGO_INPUT_H_
-#define MANGO_INPUT_H_
+#ifndef KERNEL_INPUT_H_
+#define KERNEL_INPUT_H_
 
 #include <stdint.h>
-#include <mango/queue.h>
+#include <kernel/queue.h>
 #include <mango/status.h>
 
 enum input_event_hook_flags {

include/kernel/iovec.h

@@ -0,0 +1,23 @@
+#ifndef KERNEL_IOVEC_H_
+#define KERNEL_IOVEC_H_
+
+#include <mango/types.h>
+#include <stddef.h>
+
+struct iovec_iterator {
+	const struct iovec *it_vecs;
+	size_t it_nr_vecs;
+	size_t it_vec_ptr;
+
+	virt_addr_t it_base;
+	size_t it_len;
+};
+
+extern void iovec_iterator_begin(
+	struct iovec_iterator *it,
+	const struct iovec *vecs,
+	size_t nr_vecs);
+
+extern void iovec_iterator_seek(struct iovec_iterator *it, size_t nr_bytes);
+
+#endif

include/kernel/locks.h

@@ -1,8 +1,8 @@
-#ifndef MANGO_LOCKS_H_
-#define MANGO_LOCKS_H_
+#ifndef KERNEL_LOCKS_H_
+#define KERNEL_LOCKS_H_
 
-#include <mango/compiler.h>
-#include <mango/machine/hwlock.h>
+#include <kernel/compiler.h>
+#include <kernel/machine/hwlock.h>
 
 #ifdef __cplusplus
 extern "C" {

include/kernel/memblock.h

@@ -19,12 +19,12 @@
     contributors may be used to endorse or promote products derived from this
     software without specific prior written permission.
  */
-#ifndef MANGO_MEMBLOCK_H_
-#define MANGO_MEMBLOCK_H_
+#ifndef KERNEL_MEMBLOCK_H_
+#define KERNEL_MEMBLOCK_H_
 
-#include <stddef.h>
 #include <limits.h>
-#include <mango/types.h>
+#include <kernel/types.h>
+#include <stddef.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -34,7 +34,8 @@ extern "C" {
 #define MEMBLOCK_INIT_RESERVED_REGION_COUNT 128
 
 #define __for_each_mem_range(i, type_a, type_b, p_start, p_end)                \
-	for ((i)->__idx = 0, __next_memory_region(i, type_a, type_b, p_start, p_end); \
+	for ((i)->__idx = 0,                                                   \
+	    __next_memory_region(i, type_a, type_b, p_start, p_end);           \
 	     (i)->__idx != ULLONG_MAX;                                         \
 	     __next_memory_region(i, type_a, type_b, p_start, p_end))
 
@@ -139,18 +140,24 @@ extern "C" {
      - 0x10000 -> 0x1ffff
 */
 #define for_each_free_mem_range(i, p_start, p_end)                             \
-	__for_each_mem_range(i, &memblock.memory, &memblock.reserved, p_start, p_end)
+	__for_each_mem_range(                                                  \
+		i,                                                             \
+		&memblock.memory,                                              \
+		&memblock.reserved,                                            \
+		p_start,                                                       \
+		p_end)
 
 typedef uint64_t memblock_index_t;
 
 enum memblock_region_status {
-	/* Used in memblock.memory regions, indicates that the memory region exists */
+	/* Used in memblock.memory regions, indicates that the memory region
+	 * exists */
 	MEMBLOCK_MEMORY = 0,
-	/* Used in memblock.reserved regions, indicates that the memory region was reserved
-	 * by a call to memblock_alloc() */
+	/* Used in memblock.reserved regions, indicates that the memory region
+	 * was reserved by a call to memblock_alloc() */
 	MEMBLOCK_ALLOC,
-	/* Used in memblock.reserved regions, indicates that the memory region was reserved
-	 * by a call to memblock_reserve() */
+	/* Used in memblock.reserved regions, indicates that the memory region
+	 * was reserved by a call to memblock_reserve() */
 	MEMBLOCK_RESERVED,
 };
 
@@ -176,9 +183,10 @@ struct memblock {
 	/* bounds of the memory region that can be used by memblock_alloc()
 	   both of these are virtual addresses */
 	uintptr_t m_alloc_start, m_alloc_end;
-	/* memblock assumes that all memory in the alloc zone is contiguously mapped
-	   (if paging is enabled). m_voffset is the offset that needs to be added to
-	   a given physical address to get the corresponding virtual address */
+	/* memblock assumes that all memory in the alloc zone is contiguously
+	   mapped (if paging is enabled). m_voffset is the offset that needs to
+	   be added to a given physical address to get the corresponding virtual
+	   address */
 	uintptr_t m_voffset;
 
 	struct memblock_type memory;
@@ -212,7 +220,10 @@ extern int __next_mem_range(struct memblock_iter *it);
    @param voffset the offset between the physical address of a given page and
                   its corresponding virtual address.
 */
-extern int memblock_init(uintptr_t alloc_start, uintptr_t alloc_end, uintptr_t voffset);
+extern int memblock_init(
+	uintptr_t alloc_start,
+	uintptr_t alloc_end,
+	uintptr_t voffset);
 
 /* add a region of memory to memblock.
 
@@ -234,7 +245,8 @@ extern int memblock_add(phys_addr_t base, size_t size);
    reserved memory will not be used by memblock_alloc(), and will remain
    reserved when the vm_page memory map is initialised.
 
-   @param base the physical address of the start of the memory region to reserve.
+   @param base the physical address of the start of the memory region to
+   reserve.
    @oaram size the size of the memory region to reserve in bytes.
 */
 extern int memblock_reserve(phys_addr_t base, size_t size);
@@ -310,7 +322,7 @@ extern int memblock_free_phys(phys_addr_t addr, size_t size);
 
    @param p the pointer to convert.
 */
-extern phys_addr_t memblock_virt_to_phys(void *p);
+extern phys_addr_t memblock_virt_to_phys(const void *p);
 
 /* convert a physical address returned by memblock
    to a virtual pointer.
@@ -319,9 +331,12 @@ extern phys_addr_t memblock_virt_to_phys(void *p);
 */
 extern void *memblock_phys_to_virt(phys_addr_t p);
 
-extern void __next_memory_region(struct memblock_iter *it, \
-		struct memblock_type *type_a, struct memblock_type *type_b,
-		phys_addr_t start, phys_addr_t end);
+extern void __next_memory_region(
+	struct memblock_iter *it,
+	struct memblock_type *type_a,
+	struct memblock_type *type_b,
+	phys_addr_t start,
+	phys_addr_t end);
 
 #ifdef __cplusplus
 }

include/kernel/msg.h

@@ -0,0 +1,30 @@
+#ifndef KERNEL_MSG_H_
+#define KERNEL_MSG_H_
+
+#include <kernel/btree.h>
+#include <kernel/locks.h>
+#include <mango/status.h>
+#include <mango/types.h>
+
+struct port;
+struct thread;
+
+enum kmsg_status {
+	KMSG_WAIT_RECEIVE,
+	KMSG_WAIT_REPLY,
+	KMSG_REPLY_SENT,
+};
+
+struct kmsg {
+	spin_lock_t msg_lock;
+	enum kmsg_status msg_status;
+	struct btree_node msg_node;
+	msgid_t msg_id;
+	kern_status_t msg_result;
+	struct port *msg_sender_port;
+	struct thread *msg_sender_thread;
+	const struct msg *msg_req;
+	struct msg *msg_resp;
+};
+
+#endif

include/kernel/object.h

@@ -1,16 +1,38 @@
-#ifndef MANGO_OBJECT_H_
-#define MANGO_OBJECT_H_
+#ifndef KERNEL_OBJECT_H_
+#define KERNEL_OBJECT_H_
 
-#include <mango/flags.h>
-#include <mango/locks.h>
+#include <kernel/flags.h>
+#include <kernel/locks.h>
 #include <mango/status.h>
-#include <mango/vm.h>
+#include <kernel/vm.h>
 #include <stddef.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+#define DEFINE_OBJECT_LOCK_FUNCTION(object_name, base)                         \
+	static inline void object_name##_lock(struct object_name *p)           \
+	{                                                                      \
+		object_lock(&p->base);                                         \
+	}                                                                      \
+	static inline void object_name##_unlock(struct object_name *p)         \
+	{                                                                      \
+		object_unlock(&p->base);                                       \
+	}                                                                      \
+	static inline void object_name##_lock_irqsave(                         \
+		struct object_name *p,                                         \
+		unsigned long *flags)                                          \
+	{                                                                      \
+		object_lock_irqsave(&p->base, flags);                          \
+	}                                                                      \
+	static inline void object_name##_unlock_irqrestore(                    \
+		struct object_name *p,                                         \
+		unsigned long flags)                                           \
+	{                                                                      \
+		object_unlock_irqrestore(&p->base, flags);                     \
+	}
+
 #define OBJECT_MAGIC    0xBADDCAFE
 #define OBJECT_NAME_MAX 64
 #define OBJECT_PATH_MAX 256
@@ -58,7 +80,9 @@ extern kern_status_t object_type_unregister(struct object_type *p);
 
 extern struct object *object_create(struct object_type *type);
 extern struct object *object_ref(struct object *obj);
-extern void object_deref(struct object *obj);
+extern void object_unref(struct object *obj);
+extern void object_add_handle(struct object *obj);
+extern void object_remove_handle(struct object *obj);
 extern void object_lock(struct object *obj);
 extern void object_unlock(struct object *obj);
 extern void object_lock_irqsave(struct object *obj, unsigned long *flags);

include/kernel/panic.h

@@ -0,0 +1,13 @@
+#ifndef KERNEL_PANIC_H_
+#define KERNEL_PANIC_H_
+
+#include <kernel/compiler.h>
+
+struct ml_cpu_context;
+
+#define panic(...) panic_irq(NULL, __VA_ARGS__)
+
+extern void __noreturn
+panic_irq(struct ml_cpu_context *ctx, const char *fmt, ...);
+
+#endif

include/kernel/percpu.h

@@ -1,9 +1,9 @@
-#ifndef MANGO_PERCPU_H_
-#define MANGO_PERCPU_H_
+#ifndef KERNEL_PERCPU_H_
+#define KERNEL_PERCPU_H_
 
 #include <mango/status.h>
-#include <mango/compiler.h>
-#include <mango/sched.h>
+#include <kernel/compiler.h>
+#include <kernel/sched.h>
 
 #ifdef __cplusplus
 extern "C" {

include/kernel/pmap.h

@@ -0,0 +1,81 @@
+#ifndef KERNEL_PMAP_H_
+#define KERNEL_PMAP_H_
+
+/* all the functions declared in this file are defined in arch/xyz/pmap.c */
+
+#include <kernel/machine/pmap.h>
+#include <kernel/vm.h>
+#include <mango/status.h>
+#include <stddef.h>
+
+#define PMAP_INVALID ML_PMAP_INVALID
+#define PFN(x)       ((x) >> VM_PAGE_SHIFT)
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef ml_pmap_t pmap_t;
+typedef ml_pfn_t pfn_t;
+
+enum pmap_fault_flags {
+	/* if set, the faulting page is present, and the fault is
+	 * protection-related.
+	 * if clear, the faulting page is missing, and the
+	 * fault is due to the missing page.
+	 */
+	PMAP_FAULT_PRESENT = 0x01u,
+	/* if set, the faulting page was accessed from user mode.
+	 * if clear, the faulting page was accessed from kernel mode.
+	 */
+	PMAP_FAULT_USER = 0x02u,
+	/* if set, the fault was caused by a write operation.
+	 * if clear, the faulting page was caused by a read operation.
+	 */
+	PMAP_FAULT_WRITE = 0x04u,
+	/* if set, the fault was caused while fetching an instruction from the
+	 * faulting page.
+	 */
+	PMAP_FAULT_IFETCH = 0x08u,
+	/* if set, the fault was caused by misconfigured page tables */
+	PMAP_FAULT_BADCFG = 0x10u,
+};
+
+enum pmap_flags {
+	PMAP_NORMAL = 0x00u,
+	PMAP_HUGEPAGE = 0x01u,
+};
+
+extern void pmap_bootstrap(void);
+extern pmap_t get_kernel_pmap(void);
+
+extern pmap_t pmap_create(void);
+extern void pmap_destroy(pmap_t pmap);
+extern void pmap_switch(pmap_t pmap);
+
+extern kern_status_t pmap_handle_fault(
+	virt_addr_t fault_addr,
+	enum pmap_fault_flags flags);
+
+extern kern_status_t pmap_add(
+	pmap_t pmap,
+	virt_addr_t p,
+	pfn_t pfn,
+	vm_prot_t prot,
+	enum pmap_flags flags);
+extern kern_status_t pmap_add_block(
+	pmap_t pmap,
+	virt_addr_t p,
+	pfn_t pfn,
+	size_t len,
+	vm_prot_t prot,
+	enum pmap_flags flags);
+
+extern kern_status_t pmap_remove(pmap_t pmap, virt_addr_t p);
+extern kern_status_t pmap_remove_range(pmap_t pmap, virt_addr_t p, size_t len);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

include/kernel/port.h

@@ -0,0 +1,39 @@
+#ifndef KERNEL_PORT_H_
+#define KERNEL_PORT_H_
+
+#include <kernel/object.h>
+#include <kernel/sched.h>
+
+enum port_status {
+	/* port is not connected */
+	PORT_OFFLINE = 0,
+	/* port is connected and ready to send messages */
+	PORT_READY,
+	/* port has sent a message, and is waiting for the remote to receive it
+	 */
+	PORT_SEND_BLOCKED,
+	/* port has sent a message, and the remote has received it. waiting for
+	 * the remote to reply. */
+	PORT_REPLY_BLOCKED,
+};
+
+struct port {
+	struct object p_base;
+	enum port_status p_status;
+	struct channel *p_remote;
+};
+
+extern kern_status_t port_type_init(void);
+extern struct port *port_cast(struct object *obj);
+
+extern struct port *port_create(void);
+
+extern kern_status_t port_connect(struct port *port, struct channel *remote);
+extern kern_status_t port_send_msg(
+	struct port *port,
+	const struct msg *req,
+	struct msg *resp);
+
+DEFINE_OBJECT_LOCK_FUNCTION(port, p_base)
+
+#endif

include/kernel/printk.h

@@ -0,0 +1,25 @@
+#ifndef KERNEL_PRINTK_H_
+#define KERNEL_PRINTK_H_
+
+#include <kernel/console.h>
+
+#undef TRACE
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef TRACE
+#define tracek(...) printk(__VA_ARGS__)
+#else
+#define tracek(...)
+#endif
+
+extern void early_printk_init(struct console *con);
+extern int printk(const char *format, ...);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

include/kernel/queue.h

@@ -1,7 +1,7 @@
-#ifndef MANGO_QUEUE_H_
-#define MANGO_QUEUE_H_
+#ifndef KERNEL_QUEUE_H_
+#define KERNEL_QUEUE_H_
 
-#include <mango/libc/string.h>
+#include <kernel/libc/string.h>
 #include <stdbool.h>
 
 #ifdef __cplusplus

include/kernel/ringbuffer.h

@@ -1,8 +1,8 @@
-#ifndef MANGO_RINGBUFFER_H_
-#define MANGO_RINGBUFFER_H_
+#ifndef KERNEL_RINGBUFFER_H_
+#define KERNEL_RINGBUFFER_H_
 
-#include <mango/locks.h>
-#include <mango/sched.h>
+#include <kernel/locks.h>
+#include <kernel/sched.h>
 
 struct ringbuffer {
 	unsigned char *r_buffer;

include/kernel/sched.h

@@ -1,15 +1,18 @@
-#ifndef MANGO_SCHED_H_
-#define MANGO_SCHED_H_
+#ifndef KERNEL_SCHED_H_
+#define KERNEL_SCHED_H_
 
-#include <mango/btree.h>
-#include <mango/locks.h>
-#include <mango/object.h>
-#include <mango/pmap.h>
-#include <mango/queue.h>
+#include <kernel/btree.h>
+#include <kernel/handle.h>
+#include <kernel/locks.h>
+#include <kernel/msg.h>
+#include <kernel/object.h>
+#include <kernel/pmap.h>
+#include <kernel/queue.h>
 #include <mango/status.h>
 
 #define TASK_NAME_MAX       64
 #define PRIO_MAX            32
+#define PID_MAX             99999
 #define THREAD_KSTACK_ORDER VM_PAGE_4K
 #define THREAD_MAX          65536
 
@@ -32,6 +35,7 @@
 extern "C" {
 #endif
 
+struct channel;
 struct runqueue;
 struct work_item;
 
@@ -75,13 +79,19 @@ struct task {
 	struct object t_base;
 
 	struct task *t_parent;
-	unsigned int t_id;
+	long t_id;
 	enum task_state t_state;
 	char t_name[TASK_NAME_MAX];
 
 	pmap_t t_pmap;
+	struct vm_region *t_address_space;
+	struct handle_table *t_handles;
+	struct btree b_channels;
 
 	struct btree_node t_tasklist;
+	struct queue_entry t_child_entry;
+
+	size_t t_next_thread_id;
 	struct queue t_threads;
 	struct queue t_children;
 };
@@ -100,14 +110,17 @@ struct thread {
 	cycles_t tr_quantum_cycles, tr_quantum_target;
 	cycles_t tr_total_cycles;
 
-	uintptr_t tr_sp, tr_bp;
+	virt_addr_t tr_ip, tr_sp, tr_bp;
+	virt_addr_t tr_cpu_user_sp, tr_cpu_kernel_sp;
 
 	struct runqueue *tr_rq;
+	struct kmsg tr_msg;
 
-	struct queue_entry tr_threads;
+	struct queue_entry tr_parent_entry;
 	struct queue_entry tr_rqentry;
 
 	struct vm_page *tr_kstack;
+	struct vm_object *tr_ustack;
 };
 
 struct runqueue {
@@ -175,15 +188,35 @@ extern void rq_remove_thread(struct runqueue *rq, struct thread *thr);
 extern struct runqueue *cpu_rq(unsigned int cpu);
 
 extern struct task *task_alloc(void);
+extern struct task *task_cast(struct object *obj);
+extern struct task *task_create(const char *name, size_t name_len);
 static inline struct task *task_ref(struct task *task)
 {
 	return OBJECT_CAST(struct task, t_base, object_ref(&task->t_base));
 }
-static inline void task_deref(struct task *task)
+static inline void task_unref(struct task *task)
 {
-	object_deref(&task->t_base);
+	object_unref(&task->t_base);
 }
-extern struct task *task_from_pid(unsigned int pid);
+extern kern_status_t task_add_child(struct task *parent, struct task *child);
+extern kern_status_t task_add_channel(
+	struct task *task,
+	struct channel *channel,
+	unsigned int id);
+extern struct channel *task_get_channel(struct task *task, unsigned int id);
+extern struct task *task_from_tid(tid_t id);
+extern kern_status_t task_open_handle(
+	struct task *task,
+	struct object *obj,
+	handle_flags_t flags,
+	kern_handle_t *out);
+extern kern_status_t task_resolve_handle(
+	struct task *task,
+	kern_handle_t handle,
+	struct object **out_obj,
+	handle_flags_t *out_flags);
+extern kern_status_t task_close_handle(struct task *task, kern_handle_t handle);
+extern struct thread *task_create_thread(struct task *parent);
 extern struct task *kernel_task(void);
 extern struct task *idle_task(void);
 extern cycles_t default_quantum(void);
@@ -198,21 +231,19 @@ extern void schedule_thread_on_cpu(struct thread *thr);
 extern void start_charge_period(void);
 extern void end_charge_period(void);
 
-static inline void task_lock_irqsave(struct task *task, unsigned long *flags)
-{
-	object_lock_irqsave(&task->t_base, flags);
-}
-
-static inline void task_unlock_irqrestore(
-	struct task *task,
-	unsigned long flags)
-{
-	object_unlock_irqrestore(&task->t_base, flags);
-}
+DEFINE_OBJECT_LOCK_FUNCTION(task, t_base)
 
 extern struct thread *thread_alloc(void);
-extern kern_status_t thread_init(struct thread *thr, uintptr_t ip);
+extern struct thread *thread_cast(struct object *obj);
+extern kern_status_t thread_init_kernel(struct thread *thr, virt_addr_t ip);
+extern kern_status_t thread_init_user(
+	struct thread *thr,
+	virt_addr_t ip,
+	virt_addr_t sp,
+	const uintptr_t *args,
+	size_t nr_args);
 extern int thread_priority(struct thread *thr);
+extern void thread_awaken(struct thread *thr);
 extern void idle(void);
 extern struct thread *create_kernel_thread(void (*fn)(void));
 extern struct thread *create_idle_thread(void);

include/kernel/syscall.h

@@ -0,0 +1,183 @@
+#ifndef KERNEL_SYSCALL_H_
+#define KERNEL_SYSCALL_H_
+
+#include <kernel/handle.h>
+#include <kernel/vm.h>
+#include <mango/status.h>
+#include <mango/syscall.h>
+
+#define validate_access(task, ptr, len, flags)                                 \
+	vm_region_validate_access(                                             \
+		task->t_address_space,                                         \
+		(virt_addr_t)ptr,                                              \
+		len,                                                           \
+		flags | VM_PROT_USER)
+#define validate_access_r(task, ptr, len)                                      \
+	validate_access(task, ptr, len, VM_PROT_READ | VM_PROT_USER)
+#define validate_access_w(task, ptr, len)                                      \
+	validate_access(task, ptr, len, VM_PROT_WRITE | VM_PROT_USER)
+#define validate_access_rw(task, ptr, len)                                     \
+	validate_access(                                                       \
+		task,                                                          \
+		ptr,                                                           \
+		len,                                                           \
+		VM_PROT_READ | VM_PROT_WRITE | VM_PROT_USER)
+
+extern kern_status_t sys_task_exit(int status);
+extern kern_status_t sys_task_create(
+	kern_handle_t parent_handle,
+	const char *name,
+	size_t name_len,
+	kern_handle_t *out_task,
+	kern_handle_t *out_address_space);
+extern kern_status_t sys_task_create_thread(
+	kern_handle_t task,
+	virt_addr_t ip,
+	virt_addr_t sp,
+	uintptr_t *args,
+	size_t nr_args,
+	kern_handle_t *out_thread);
+
+extern kern_status_t sys_thread_start(kern_handle_t thread);
+
+extern kern_status_t sys_vm_object_create(
+	const char *name,
+	size_t name_len,
+	size_t data_len,
+	vm_prot_t prot,
+	kern_handle_t *out);
+extern kern_status_t sys_vm_object_read(
+	kern_handle_t object,
+	void *dst,
+	off_t offset,
+	size_t count,
+	size_t *nr_read);
+extern kern_status_t sys_vm_object_write(
+	kern_handle_t object,
+	const void *src,
+	off_t offset,
+	size_t count,
+	size_t *nr_written);
+extern kern_status_t sys_vm_object_copy(
+	kern_handle_t dst,
+	off_t dst_offset,
+	kern_handle_t src,
+	off_t src_offset,
+	size_t count,
+	size_t *nr_copied);
+
+extern kern_status_t sys_vm_region_create(
+	kern_handle_t parent,
+	const char *name,
+	size_t name_len,
+	off_t offset,
+	size_t region_len,
+	vm_prot_t prot,
+	kern_handle_t *out,
+	virt_addr_t *out_base_address);
+extern kern_status_t sys_vm_region_read(
+	kern_handle_t region,
+	void *dst,
+	off_t offset,
+	size_t count,
+	size_t *nr_read);
+extern kern_status_t sys_vm_region_write(
+	kern_handle_t region,
+	const void *src,
+	off_t offset,
+	size_t count,
+	size_t *nr_read);
+extern kern_status_t sys_vm_region_map_absolute(
+	kern_handle_t region,
+	virt_addr_t map_address,
+	kern_handle_t object,
+	off_t object_offset,
+	size_t length,
+	vm_prot_t prot,
+	virt_addr_t *out_base_address);
+extern kern_status_t sys_vm_region_map_relative(
+	kern_handle_t region,
+	off_t region_offset,
+	kern_handle_t object,
+	off_t object_offset,
+	size_t length,
+	vm_prot_t prot,
+	virt_addr_t *out_base_address);
+extern kern_status_t sys_vm_region_unmap_absolute(
+	kern_handle_t region,
+	virt_addr_t address,
+	size_t length);
+extern kern_status_t sys_vm_region_unmap_relative(
+	kern_handle_t region,
+	off_t offset,
+	size_t length);
+
+extern kern_status_t sys_kern_log(const char *s);
+extern kern_status_t sys_kern_handle_close(kern_handle_t handle);
+extern kern_status_t sys_kern_config_get(
+	kern_config_key_t key,
+	void *ptr,
+	size_t len);
+extern kern_status_t sys_kern_config_set(
+	kern_config_key_t key,
+	const void *ptr,
+	size_t len);
+
+extern kern_status_t sys_channel_create(
+	unsigned int id,
+	channel_flags_t flags,
+	kern_handle_t *out);
+extern kern_status_t sys_port_create(kern_handle_t *out);
+extern kern_status_t sys_port_connect(
+	kern_handle_t port,
+	tid_t task_id,
+	unsigned int channel_id);
+extern kern_status_t sys_port_disconnect(kern_handle_t port);
+
+extern kern_status_t sys_msg_send(
+	kern_handle_t port,
+	msg_flags_t flags,
+	const struct msg *req,
+	struct msg *resp);
+
+extern kern_status_t sys_msg_recv(
+	kern_handle_t channel,
+	msg_flags_t flags,
+	msgid_t *out_id,
+	struct msg *out_msg);
+
+extern kern_status_t sys_msg_reply(
+	kern_handle_t channel,
+	msg_flags_t flags,
+	msgid_t id,
+	const struct msg *reply);
+
+extern kern_status_t sys_msg_read(
+	kern_handle_t channel,
+	msgid_t id,
+	size_t offset,
+	struct iovec *out,
+	size_t nr_out);
+extern kern_status_t sys_msg_read_handles(
+	kern_handle_t channel,
+	msgid_t id,
+	size_t offset,
+	struct handle_list *out,
+	size_t nr_out);
+
+extern kern_status_t sys_msg_write(
+	kern_handle_t channel,
+	msgid_t id,
+	size_t offset,
+	const struct iovec *in,
+	size_t nr_in);
+extern kern_status_t sys_msg_write_handles(
+	kern_handle_t channel,
+	msgid_t id,
+	size_t offset,
+	const struct handle_list *in,
+	size_t nr_in);
+
+extern virt_addr_t syscall_get_function(unsigned int sysid);
+
+#endif

include/kernel/test.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_TEST_H_
-#define MANGO_TEST_H_
+#ifndef KERNEL_TEST_H_
+#define KERNEL_TEST_H_
 
 #ifdef __cplusplus
 extern "C" {

include/kernel/types.h

@@ -0,0 +1,18 @@
+#ifndef KERNEL_TYPES_H_
+#define KERNEL_TYPES_H_
+
+#include <mango/types.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#define CYCLES_MAX UINT64_MAX
+
+typedef uint64_t cycles_t;
+typedef uint64_t sectors_t;
+
+struct boot_module {
+	phys_addr_t mod_base;
+	size_t mod_size;
+};
+
+#endif

include/kernel/util.h

@@ -1,9 +1,10 @@
-#ifndef MANGO_UTIL_H_
-#define MANGO_UTIL_H_
+#ifndef KERNEL_UTIL_H_
+#define KERNEL_UTIL_H_
 
+#include <mango/types.h>
+#include <stdbool.h>
 #include <stddef.h>
 #include <stdint.h>
-#include <stdbool.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -15,13 +16,20 @@ extern "C" {
 
 extern uint64_t hash_string(const char *s);
 extern void data_size_to_string(size_t value, char *out, size_t outsz);
-static inline bool power_of_2(size_t x) { return (x > 0 && (x & (x - 1)) == 0); }
-static inline unsigned long long div64_pow2(unsigned long long x, unsigned long long y)
+static inline bool power_of_2(size_t x)
+{
+	return (x > 0 && (x & (x - 1)) == 0);
+}
+static inline unsigned long long div64_pow2(
+	unsigned long long x,
+	unsigned long long y)
 {
 	return x >> (__builtin_ctz(y));
 }
 
-static inline unsigned long long absdiff64(unsigned long long x, unsigned long long y)
+static inline unsigned long long absdiff64(
+	unsigned long long x,
+	unsigned long long y)
 {
 	return x < y ? y - x : x - y;
 }
@@ -53,6 +61,8 @@ extern uint64_t host_to_little_u64(uint64_t v);
 extern uint64_t big_to_host_u64(uint64_t v);
 extern uint64_t little_to_host_u64(uint64_t v);
 
+extern bool fill_random(void *buffer, unsigned int size);
+
 #ifdef __cplusplus
 }
 #endif

include/kernel/vm-object.h

@@ -0,0 +1,91 @@
+#ifndef KERNEL_VM_OBJECT_H_
+#define KERNEL_VM_OBJECT_H_
+
+#include <kernel/locks.h>
+#include <kernel/object.h>
+
+#define VM_OBJECT_NAME_MAX 64
+
+enum vm_object_flags {
+	/* the memory behind this vm-object wasn't allocated by us, and
+	 * therefore shouldn't be freed by us */
+	VMO_IN_PLACE = 0x01u,
+};
+
+struct vm_object {
+	struct object vo_base;
+
+	char vo_name[VM_OBJECT_NAME_MAX];
+	enum vm_object_flags vo_flags;
+
+	/* queue of struct vm_region_mapping */
+	struct queue vo_mappings;
+
+	/* memory protection flags. mappings of this vm_object can only use
+	 * a subset of the flags set in this mask. */
+	vm_prot_t vo_prot;
+
+	/* btree of vm_pages that have been allocated to this vm_object.
+	 * vm_page->p_vmo_offset and the size of each page is the bst key. */
+	struct btree vo_pages;
+	/* total length of the vm_object in bytes. */
+	size_t vo_size;
+};
+
+extern kern_status_t vm_object_type_init(void);
+extern struct vm_object *vm_object_cast(struct object *obj);
+
+/* create a vm_object with the specified length in bytes and protection flags.
+ * the length will be automatically rounded up to the nearest vm_object page
+ * order size. the actual page frames themselves won't be allocated until
+ * they are mapped and accessed. */
+extern struct vm_object *vm_object_create(
+	const char *name,
+	size_t name_len,
+	size_t data_len,
+	vm_prot_t prot);
+
+/* create a vm_object that represents the specified range of physical memory.
+ * the length will be automatically rounded up to the nearest vm_object page
+ * order size.
+ * NOTE this function assumes that the physical memory has already been
+ * reserved, and is not in use by any other kernel component. */
+extern struct vm_object *vm_object_create_in_place(
+	const char *name,
+	size_t name_len,
+	phys_addr_t base,
+	size_t data_len,
+	vm_prot_t prot);
+
+extern struct vm_page *vm_object_get_page(
+	const struct vm_object *vo,
+	off_t offset);
+
+extern struct vm_page *vm_object_alloc_page(
+	struct vm_object *vo,
+	off_t offset,
+	enum vm_page_order size);
+
+extern kern_status_t vm_object_read(
+	struct vm_object *vo,
+	void *dst,
+	off_t offset,
+	size_t count,
+	size_t *nr_read);
+extern kern_status_t vm_object_write(
+	struct vm_object *vo,
+	const void *dst,
+	off_t offset,
+	size_t count,
+	size_t *nr_written);
+extern kern_status_t vm_object_copy(
+	struct vm_object *dst,
+	off_t dst_offset,
+	struct vm_object *src,
+	off_t src_offset,
+	size_t count,
+	size_t *nr_copied);
+
+DEFINE_OBJECT_LOCK_FUNCTION(vm_object, vo_base)
+
+#endif

include/kernel/vm-region.h

@@ -0,0 +1,146 @@
+#ifndef KERNEL_VM_REGION_H_
+#define KERNEL_VM_REGION_H_
+
+#include <kernel/object.h>
+#include <kernel/pmap.h>
+#include <kernel/vm.h>
+
+#define VM_REGION_NAME_MAX 64
+#define VM_REGION_COPY_ALL ((size_t)-1)
+
+struct vm_region;
+struct vm_object;
+
+enum vm_region_entry_type {
+	VM_REGION_ENTRY_NONE = 0,
+	VM_REGION_ENTRY_REGION,
+	VM_REGION_ENTRY_MAPPING,
+};
+
+struct vm_region_entry {
+	struct btree_node e_node;
+	struct vm_region_entry *e_parent;
+	enum vm_region_entry_type e_type;
+	/* offset in bytes of this entry within its immediate parent. */
+	off_t e_offset;
+	/* size of the entry in bytes */
+	size_t e_size;
+};
+
+struct vm_region_mapping {
+	struct vm_region_entry m_entry;
+	struct vm_object *m_object;
+
+	/* used to link to  vm_object->vo_mappings */
+	struct queue_entry m_object_entry;
+
+	vm_prot_t m_prot;
+	/* offset in bytes to the start of the object data that was mapped */
+	off_t m_object_offset;
+};
+
+struct vm_region {
+	struct object vr_base;
+	struct vm_region_entry vr_entry;
+
+	char vr_name[VM_REGION_NAME_MAX];
+
+	/* btree of struct vm_region_entry.
+	 * sibling entries cannot overlap each other, and child entries must
+	 * be entirely contained within their immediate parent entry. */
+	struct btree vr_entries;
+
+	/* memory protection restriction mask.
+	 * any mapping in this region, or any of its children, cannot use
+	 * protection flags that are not set in this mask.
+	 * for example, if VM_PROT_EXEC is /not/ set here, no mapping
+	 * can be created in this region or any child region with VM_PROT_EXEC
+	 * set. */
+	vm_prot_t vr_prot;
+
+	/* the physical address space in which mappings in this region (and
+	 * its children) are created */
+	pmap_t vr_pmap;
+};
+
+extern kern_status_t vm_region_type_init(void);
+extern struct vm_region *vm_region_cast(struct object *obj);
+
+/* create a new vm-region, optionally within a parent region.
+ * `offset` is the byte offset within the parent region where the new region
+ * should start.
+ * if no parent is specified, `offset` is the absolute virtual address of the
+ * start of the region.
+ * in both cases, `len` is the length of the new region in bytes. */
+extern kern_status_t vm_region_create(
+	struct vm_region *parent,
+	const char *name,
+	size_t name_len,
+	off_t offset,
+	size_t region_len,
+	vm_prot_t prot,
+	struct vm_region **out);
+
+/* map a vm-object into a vm-region.
+ * [region_offset,length] must fall within exactly one region, and cannot span
+ * multiple sibling regions.
+ * if [region_offset,length] falls within a child region, the map operation
+ * will be transparently redirected to the relevant region.
+ * `prot` must be allowed both by the region into which the mapping is being
+ * created AND the vm-object being mapped. */
+extern kern_status_t vm_region_map_object(
+	struct vm_region *region,
+	off_t region_offset,
+	struct vm_object *object,
+	off_t object_offset,
+	size_t length,
+	vm_prot_t prot,
+	virt_addr_t *out);
+
+extern kern_status_t vm_region_unmap(
+	struct vm_region *region,
+	off_t region_offset,
+	size_t length);
+
+extern bool vm_region_validate_access(
+	struct vm_region *region,
+	off_t offset,
+	size_t len,
+	vm_prot_t prot);
+
+/* find the mapping corresponding to the given virtual address, and page-in the
+ * necessary vm_page to allow the memory access to succeed. if the relevant
+ * vm-object page hasn't been allocated yet, it will be allocated here. */
+extern kern_status_t vm_region_demand_map(
+	struct vm_region *region,
+	virt_addr_t addr,
+	enum pmap_fault_flags flags);
+
+/* get the absolute base virtual address of a region within its
+ * parent/ancestors. */
+extern virt_addr_t vm_region_get_base_address(const struct vm_region *region);
+
+extern void vm_region_dump(struct vm_region *region);
+
+extern kern_status_t vm_region_memmove(
+	struct vm_region *dest_region,
+	virt_addr_t dest_ptr,
+	struct vm_region *src_region,
+	virt_addr_t src_ptr,
+	size_t count,
+	size_t *nr_moved);
+
+extern kern_status_t vm_region_memmove_v(
+	struct vm_region *dest_region,
+	size_t dest_offset,
+	struct iovec *dest,
+	size_t nr_dest,
+	struct vm_region *src_region,
+	size_t src_offset,
+	const struct iovec *src,
+	size_t nr_src,
+	size_t bytes_to_move);
+
+DEFINE_OBJECT_LOCK_FUNCTION(vm_region, vr_base)
+
+#endif

include/kernel/vm.h

@@ -1,14 +1,14 @@
-#ifndef MANGO_VM_H_
-#define MANGO_VM_H_
+#ifndef KERNEL_VM_H_
+#define KERNEL_VM_H_
 
-#include <stddef.h>
-#include <mango/types.h>
+#include <kernel/bitmap.h>
+#include <kernel/btree.h>
+#include <kernel/locks.h>
+#include <kernel/machine/vm.h>
+#include <kernel/queue.h>
+#include <kernel/types.h>
 #include <mango/status.h>
-#include <mango/queue.h>
-#include <mango/btree.h>
-#include <mango/bitmap.h>
-#include <mango/locks.h>
-#include <mango/machine/vm.h>
+#include <stddef.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -33,7 +33,8 @@ struct bcache;
 #define VM_CHECK_ALIGN(p, mask) ((((p) & (mask)) == (p)) ? 1 : 0)
 
 #define VM_CACHE_INITIALISED(c) ((c)->c_obj_count != 0)
-#define VM_PAGE_IS_FREE(pg) (((pg)->p_flags & (VM_PAGE_RESERVED | VM_PAGE_ALLOC)) == 0)
+#define VM_PAGE_IS_FREE(pg)                                                    \
+	(((pg)->p_flags & (VM_PAGE_RESERVED | VM_PAGE_ALLOC)) == 0)
 
 #define vm_page_foreach(pg, i)                                                 \
 	for (struct vm_page *i = (pg); i; i = vm_page_get_next_tail(i))
@@ -41,32 +42,19 @@ struct bcache;
 typedef phys_addr_t vm_alignment_t;
 typedef unsigned int vm_node_id_t;
 
-struct vm_object {
-	unsigned int reserved;
-};
-
 enum vm_model {
 	VM_MODEL_FLAT = 1,
 	VM_MODEL_SPARSE,
 };
 
-enum vm_prot {
-	VM_PROT_READ    = 0x01u,
-	VM_PROT_WRITE   = 0x02u,
-	VM_PROT_EXEC    = 0x04u,
-	VM_PROT_USER    = 0x08u,
-	VM_PROT_SVR     = 0x10u,
-	VM_PROT_NOCACHE = 0x20u,
-};
-
 enum vm_flags {
 	VM_NORMAL = 0x00u,
 	VM_GET_DMA = 0x01u,
 };
 
 enum vm_zone_id {
-	/* NOTE that these are used as indices into the node_zones array in vm/zone.c
-	   they need to be continuous, and must start at 0!
+	/* NOTE that these are used as indices into the node_zones array in
+	   vm/zone.c they need to be continuous, and must start at 0!
 
 	   not all of these zones are implemented for every architecture. */
 	VM_ZONE_DMA = 0u,
@@ -108,8 +96,8 @@ enum vm_page_order {
 };
 
 enum vm_page_flags {
-	/* page is reserved (probably by a call to memblock_reserve()) and cannot be
-	   returned by any allocation function */
+	/* page is reserved (probably by a call to memblock_reserve()) and
+	   cannot be returned by any allocation function */
 	VM_PAGE_RESERVED = 0x01u,
 	/* page has been allocated by a zone's buddy allocator, and is in-use */
 	VM_PAGE_ALLOC = 0x02u,
@@ -117,7 +105,8 @@ enum vm_page_flags {
 	VM_PAGE_HEAD = 0x04u,
 	/* page is part of a huge-page */
 	VM_PAGE_HUGE = 0x08u,
-	/* page is holding cached data from secondary storage, and can be freed if necessary (and not dirty). */
+	/* page is holding cached data from secondary storage, and can be freed
+	 * if necessary (and not dirty). */
 	VM_PAGE_CACHE = 0x10u,
 };
 
@@ -151,12 +140,6 @@ struct vm_pg_data {
 	struct vm_zone pg_zones[VM_MAX_ZONES];
 };
 
-struct vm_region {
-	enum vm_memory_region_status r_status;
-	phys_addr_t r_base;
-	phys_addr_t r_limit;
-};
-
 struct vm_cache {
 	const char *c_name;
 	enum vm_cache_flags c_flags;
@@ -204,7 +187,7 @@ struct vm_slab {
 	     - s_freelist[s_free] should be set to the previous value of s_free.
 	   this is commented as it as flexible arrays are not supported in c++.
 	   */
-	//unsigned int s_freelist[];
+	// unsigned int s_freelist[];
 };
 
 struct vm_page {
@@ -231,20 +214,25 @@ struct vm_page {
 	/* multi-purpose list/tree entry.
 	   the owner of the page can decide what to do with this.
 	   some examples:
-	     - the buddy allocator uses this to maintain its per-zone free-page lists.
-	     - the block cache uses this to maintain a tree of pages keyed by block number.
+	     - the buddy allocator uses this to maintain its per-zone free-page
+	       lists.
+	     - vm_object uses it to maintain a btree of allocated pages keyed
+	       by offset/size.
+	     - the block cache uses this to maintain a tree of pages keyed by
+	       block number.
 	*/
 	union {
 		struct queue_entry p_list;
 		struct btree_node p_bnode;
 
-		/* btree_node contains three pointers, so provide three pointer-sized integers for
-		   use if p_bnode isn't needed. */
+		/* btree_node contains three pointers, so provide three
+		   pointer-sized integers for use if p_bnode isn't needed. */
 		uintptr_t priv1[3];
 	};
 
 	union {
-		/* used by bcache when sector size is < page size. bitmap of present/missing sectors */
+		/* used by bcache when sector size is < page size. bitmap of
+		 * present/missing sectors */
 		DECLARE_BITMAP(p_blockbits, VM_MAX_SECTORS_PER_PAGE);
 		uint32_t p_priv2;
 	};
@@ -252,10 +240,12 @@ struct vm_page {
 	union {
 		/* sector address, used by bcache */
 		sectors_t p_blockid;
+		/* offset of this page within the vm_object it is a part of */
+		off_t p_vmo_offset;
 
 		uint32_t p_priv3[2];
 	};
-} __attribute__((aligned(2 * sizeof(unsigned long))));
+} __aligned(2 * sizeof(unsigned long));
 
 /* represents a sector of memory, containing its own array of vm_pages.
    this struct is used under the sparse memory model, instead of the
@@ -272,39 +262,58 @@ struct vm_sector {
 	struct vm_page *s_pages;
 };
 
-extern kern_status_t vm_bootstrap(const struct vm_zone_descriptor *zones, size_t nr_zones);
+extern kern_status_t vm_bootstrap(
+	const struct vm_zone_descriptor *zones,
+	size_t nr_zones);
 extern enum vm_model vm_memory_model(void);
 extern void vm_set_memory_model(enum vm_model model);
 
 extern struct vm_pg_data *vm_pg_data_get(vm_node_id_t node);
 
-extern phys_addr_t vm_virt_to_phys(void *p);
+extern phys_addr_t vm_virt_to_phys(const void *p);
 extern void *vm_phys_to_virt(phys_addr_t p);
 
-extern void vm_page_init_array();
+extern size_t vm_page_order_to_bytes(enum vm_page_order order);
+extern size_t vm_page_order_to_pages(enum vm_page_order order);
+extern vm_alignment_t vm_page_order_to_alignment(enum vm_page_order order);
+extern void vm_page_init_array(void);
 extern struct vm_page *vm_page_get(phys_addr_t addr);
 extern phys_addr_t vm_page_get_paddr(struct vm_page *pg);
 extern struct vm_zone *vm_page_get_zone(struct vm_page *pg);
 extern void *vm_page_get_vaddr(struct vm_page *pg);
 extern size_t vm_page_get_pfn(struct vm_page *pg);
-extern size_t vm_page_order_to_bytes(enum vm_page_order order);
-extern size_t vm_page_order_to_pages(enum vm_page_order order);
-extern vm_alignment_t vm_page_order_to_alignment(enum vm_page_order order);
-extern struct vm_page *vm_page_alloc(enum vm_page_order order, enum vm_flags flags);
+static inline size_t vm_page_get_size_bytes(const struct vm_page *pg)
+{
+	return vm_page_order_to_bytes(pg->p_order);
+}
+extern struct vm_page *vm_page_alloc(
+	enum vm_page_order order,
+	enum vm_flags flags);
 extern void vm_page_free(struct vm_page *pg);
 
-extern int vm_page_split(struct vm_page *pg, struct vm_page **a, struct vm_page **b);
+extern int vm_page_split(
+	struct vm_page *pg,
+	struct vm_page **a,
+	struct vm_page **b);
 extern struct vm_page *vm_page_merge(struct vm_page *a, struct vm_page *b);
 extern struct vm_page *vm_page_get_buddy(struct vm_page *pg);
 extern struct vm_page *vm_page_get_next_tail(struct vm_page *pg);
 
 extern size_t vm_bytes_to_pages(size_t bytes);
 
-extern void vm_zone_init(struct vm_zone *z, const struct vm_zone_descriptor *zone_info);
-extern struct vm_page *vm_zone_alloc_page(struct vm_zone *z, enum vm_page_order order, enum vm_flags flags);
+extern void vm_zone_init(
+	struct vm_zone *z,
+	const struct vm_zone_descriptor *zone_info);
+extern struct vm_page *vm_zone_alloc_page(
+	struct vm_zone *z,
+	enum vm_page_order order,
+	enum vm_flags flags);
 extern void vm_zone_free_page(struct vm_zone *z, struct vm_page *pg);
 
-extern struct vm_cache *vm_cache_create(const char *name, size_t objsz, enum vm_cache_flags flags);
+extern struct vm_cache *vm_cache_create(
+	const char *name,
+	size_t objsz,
+	enum vm_cache_flags flags);
 extern void vm_cache_init(struct vm_cache *cache);
 extern void vm_cache_destroy(struct vm_cache *cache);
 extern void *vm_cache_alloc(struct vm_cache *cache, enum vm_flags flags);
@@ -330,7 +339,10 @@ extern size_t vm_page_get_pfn_sparse(struct vm_page *pg);
 #endif
 
 #ifdef __cplusplus
-inline void *operator new(size_t count, void *p) { return p; }
+inline void *operator new(size_t count, void *p)
+{
+	return p;
+}
 
 #define kmalloc_object(objtype, flags, ...)                                    \
 	__extension__({                                                        \

include/mango/panic.h

@@ -1,12 +0,0 @@
-#ifndef MANGO_PANIC_H_
-#define MANGO_PANIC_H_
-
-#include <mango/compiler.h>
-
-struct cpu_context;
-
-#define panic(...) panic_irq(NULL, __VA_ARGS__)
-
-extern void __noreturn panic_irq(struct cpu_context *ctx, const char *fmt, ...);
-
-#endif

include/mango/pmap.h

@@ -1,42 +0,0 @@
-#ifndef MANGO_PMAP_H_
-#define MANGO_PMAP_H_
-
-/* all the functions declared in this file are defined in arch/xyz/pmap.c */
-
-#include <mango/vm.h>
-#include <mango/status.h>
-#include <mango/machine/pmap.h>
-#include <stddef.h>
-
-#define PFN(x) ((x) >> VM_PAGE_SHIFT)
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef ml_pmap_t pmap_t;
-typedef ml_pfn_t pfn_t;
-
-enum pmap_flags {
-	PMAP_NORMAL = 0x00u,
-	PMAP_HUGEPAGE = 0x01u,
-};
-
-extern void pmap_bootstrap(void);
-extern pmap_t get_kernel_pmap(void);
-
-extern pmap_t pmap_create(void);
-extern void pmap_destroy(pmap_t pmap);
-extern void pmap_switch(pmap_t pmap);
-
-extern kern_status_t pmap_add(pmap_t pmap, void *p, pfn_t pfn, enum vm_prot prot, enum pmap_flags flags);
-extern kern_status_t pmap_add_block(pmap_t pmap, void *p, pfn_t pfn, size_t len, enum vm_prot prot, enum pmap_flags flags);
-
-extern kern_status_t pmap_remove(pmap_t pmap, void *p);
-extern kern_status_t pmap_remove_range(pmap_t pmap, void *p, size_t len);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

include/mango/printk.h

@@ -1,17 +0,0 @@
-#ifndef MANGO_PRINTK_H_
-#define MANGO_PRINTK_H_
-
-#include <mango/console.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-extern void early_printk_init(struct console *con);
-extern int printk(const char *format, ...);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

include/mango/types.h

@@ -1,14 +0,0 @@
-#ifndef MANGO_TYPES_H_
-#define MANGO_TYPES_H_
-
-#include <stdint.h>
-
-#define CYCLES_MAX UINT64_MAX
-
-typedef uintptr_t phys_addr_t;
-typedef uint64_t cycles_t;
-typedef uint64_t sectors_t;
-
-typedef unsigned int umode_t;
-
-#endif

init/init.c

@@ -1,5 +1,4 @@
-#include <mango/init.h>
-
+#include <kernel/init.h>
 
 int do_initcalls(void)
 {

init/main.c

@@ -1,15 +1,20 @@
-#include <mango/arg.h>
-#include <mango/clock.h>
-#include <mango/cpu.h>
-#include <mango/init.h>
-#include <mango/input.h>
-#include <mango/libc/stdio.h>
-#include <mango/machine/init.h>
-#include <mango/object.h>
-#include <mango/panic.h>
-#include <mango/printk.h>
-#include <mango/sched.h>
-#include <mango/test.h>
+#include <kernel/arg.h>
+#include <kernel/bsp.h>
+#include <kernel/channel.h>
+#include <kernel/clock.h>
+#include <kernel/cpu.h>
+#include <kernel/handle.h>
+#include <kernel/init.h>
+#include <kernel/input.h>
+#include <kernel/libc/stdio.h>
+#include <kernel/machine/init.h>
+#include <kernel/object.h>
+#include <kernel/panic.h>
+#include <kernel/port.h>
+#include <kernel/printk.h>
+#include <kernel/sched.h>
+#include <kernel/test.h>
+#include <kernel/vm-object.h>
 #include <stdint.h>
 
 extern unsigned long get_rflags(void);
@@ -23,14 +28,16 @@ void print_kernel_banner(void)
 
 static void hang(void)
 {
-	struct task *self = current_task();
-	struct thread *thread = current_thread();
+	// struct task *self = current_task();
+	// struct thread *thread = current_thread();
 
 	while (1) {
-		printk("[cpu %u, task %u, thread %u]: tick",
+#if 0
+		printk("[cpu %u, task %ld, thread %u]: tick",
 		       this_cpu(),
 		       self->t_id,
 		       thread->tr_id);
+#endif
 		milli_sleep(2000);
 	}
 }
@@ -67,9 +74,43 @@ void kernel_init(uintptr_t arg)
 {
 	ml_init(arg);
 
-	printk("kernel_init() running on processor %u", this_cpu());
+	port_type_init();
+	channel_type_init();
 
-	create_kernel_thread(background_thread);
+	struct boot_module bsp_image = {0};
+	bsp_get_location(&bsp_image);
+
+	tracek("kernel_init() running on processor %u", this_cpu());
+
+	if (!bsp_image.mod_base) {
+		printk("FATAL: no bsp image specified");
+		hang();
+	}
+
+	tracek("bsp image at [0x%llx-0x%llx]",
+	       bsp_image.mod_base,
+	       bsp_image.mod_base + bsp_image.mod_size);
+
+	struct bsp bsp;
+	kern_status_t status = bsp_load(&bsp, &bsp_image);
+	if (status != KERN_OK) {
+		printk("FATAL: bsp image is corrupt/invalid");
+		hang();
+	}
+
+	tracek("bsp image loaded. text=[%06llx-%06llx], data=[%06llx-%06llx], "
+	       "entry=%06llx, vmo=%p",
+	       bsp.bsp_trailer.bsp_text_vaddr,
+	       bsp.bsp_trailer.bsp_text_size + bsp.bsp_trailer.bsp_text_vaddr,
+	       bsp.bsp_trailer.bsp_data_vaddr,
+	       bsp.bsp_trailer.bsp_data_size + bsp.bsp_trailer.bsp_data_vaddr,
+	       bsp.bsp_trailer.bsp_exec_entry,
+	       bsp.bsp_vmo);
+
+	struct task *bootstrap_task = task_create("bootstrap", 9);
+	tracek("created bootstrap task (pid=%u)", bootstrap_task->t_id);
+
+	bsp_launch_async(&bsp, bootstrap_task);
 
 	hang();
 }

kernel/arg.c

@@ -1,6 +1,6 @@
-#include <mango/arg.h>
-#include <mango/libc/string.h>
-#include <mango/libc/ctype.h>
+#include <kernel/arg.h>
+#include <kernel/libc/string.h>
+#include <kernel/libc/ctype.h>
 
 static char g_cmdline[CMDLINE_MAX + 1] = {0};
 

kernel/bsp.c

@@ -0,0 +1,244 @@
+#include <kernel/bsp.h>
+#include <kernel/handle.h>
+#include <kernel/printk.h>
+#include <kernel/sched.h>
+#include <kernel/util.h>
+#include <kernel/vm-object.h>
+#include <kernel/vm-region.h>
+
+#define BOOTSTRAP_STACK_SIZE 0x10000
+
+static struct boot_module bsp_location = {0};
+
+void bsp_set_location(const struct boot_module *mod)
+{
+	memcpy(&bsp_location, mod, sizeof bsp_location);
+}
+
+void bsp_get_location(struct boot_module *out)
+{
+	memcpy(out, &bsp_location, sizeof bsp_location);
+}
+
+kern_status_t bsp_load(struct bsp *bsp, const struct boot_module *mod)
+{
+	size_t trailer_offset = mod->mod_size - sizeof(struct bsp_trailer);
+	const char *p = vm_phys_to_virt(mod->mod_base);
+	const struct bsp_trailer *trailer_bigendian
+		= (const struct bsp_trailer *)(p + trailer_offset);
+
+	bsp->bsp_trailer.bsp_magic
+		= big_to_host_u32(trailer_bigendian->bsp_magic);
+	bsp->bsp_trailer.bsp_fs_offset
+		= big_to_host_u64(trailer_bigendian->bsp_fs_offset);
+	bsp->bsp_trailer.bsp_fs_len
+		= big_to_host_u32(trailer_bigendian->bsp_fs_len);
+	bsp->bsp_trailer.bsp_exec_offset
+		= big_to_host_u64(trailer_bigendian->bsp_exec_offset);
+	bsp->bsp_trailer.bsp_exec_len
+		= big_to_host_u32(trailer_bigendian->bsp_exec_len);
+	bsp->bsp_trailer.bsp_text_faddr
+		= big_to_host_u64(trailer_bigendian->bsp_text_faddr);
+	bsp->bsp_trailer.bsp_text_vaddr
+		= big_to_host_u64(trailer_bigendian->bsp_text_vaddr);
+	bsp->bsp_trailer.bsp_text_size
+		= big_to_host_u64(trailer_bigendian->bsp_text_size);
+	bsp->bsp_trailer.bsp_data_faddr
+		= big_to_host_u64(trailer_bigendian->bsp_data_faddr);
+	bsp->bsp_trailer.bsp_data_vaddr
+		= big_to_host_u64(trailer_bigendian->bsp_data_vaddr);
+	bsp->bsp_trailer.bsp_data_size
+		= big_to_host_u64(trailer_bigendian->bsp_data_size);
+	bsp->bsp_trailer.bsp_exec_entry
+		= big_to_host_u64(trailer_bigendian->bsp_exec_entry);
+
+	if (bsp->bsp_trailer.bsp_magic != BSP_MAGIC) {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	bsp->bsp_vmo = vm_object_create_in_place(
+		"bsp",
+		3,
+		mod->mod_base,
+		mod->mod_size,
+		VM_PROT_READ | VM_PROT_EXEC | VM_PROT_USER);
+	if (!bsp->bsp_vmo) {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	return KERN_OK;
+}
+
+static kern_status_t map_executable(
+	struct bsp *bsp,
+	struct task *task,
+	virt_addr_t *entry)
+{
+	kern_status_t status = KERN_OK;
+	size_t exec_size = 0;
+	if (bsp->bsp_trailer.bsp_text_vaddr > bsp->bsp_trailer.bsp_data_vaddr) {
+		exec_size = bsp->bsp_trailer.bsp_text_vaddr
+		          + bsp->bsp_trailer.bsp_text_size;
+	} else {
+		exec_size = bsp->bsp_trailer.bsp_data_vaddr
+		          + bsp->bsp_trailer.bsp_data_size;
+	}
+
+	struct vm_region *region;
+	status = vm_region_create(
+		task->t_address_space,
+		"exec",
+		4,
+		VM_REGION_ANY_OFFSET,
+		exec_size,
+		VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXEC | VM_PROT_USER,
+		&region);
+	if (status != KERN_OK) {
+		return status;
+	}
+
+	struct vm_object *data = vm_object_create(
+		".data",
+		5,
+		bsp->bsp_trailer.bsp_data_size,
+		VM_PROT_READ | VM_PROT_WRITE | VM_PROT_USER);
+	if (!data) {
+		return KERN_NO_MEMORY;
+	}
+
+	virt_addr_t text_base = 0, data_base = 0;
+
+	off_t text_foffset = bsp->bsp_trailer.bsp_exec_offset
+	                   + bsp->bsp_trailer.bsp_text_faddr;
+	off_t data_foffset = 0;
+	off_t text_voffset = bsp->bsp_trailer.bsp_text_vaddr;
+	off_t data_voffset = bsp->bsp_trailer.bsp_data_vaddr;
+
+#if 0
+	size_t tmp = 0;
+	status = vm_object_copy(
+		data,
+		0,
+		bsp->bsp_vmo,
+		bsp->bsp_trailer.bsp_data_faddr,
+		bsp->bsp_trailer.bsp_data_size,
+		&tmp);
+
+	tracek("read %zuB of data from executable", tmp);
+#endif
+
+	tracek("text_foffset=%06llx, data_foffset=%06llx",
+	       text_foffset,
+	       data_foffset);
+	tracek("text_voffset=%08llx, data_voffset=%08llx",
+	       text_voffset,
+	       data_voffset);
+
+	status = vm_region_map_object(
+		region,
+		text_voffset,
+		bsp->bsp_vmo,
+		text_foffset,
+		bsp->bsp_trailer.bsp_text_size,
+		VM_PROT_READ | VM_PROT_EXEC | VM_PROT_USER,
+		&text_base);
+	if (status != KERN_OK) {
+		return status;
+	}
+
+	status = vm_region_map_object(
+		region,
+		data_voffset,
+		data,
+		data_foffset,
+		bsp->bsp_trailer.bsp_data_size,
+		VM_PROT_READ | VM_PROT_WRITE | VM_PROT_USER,
+		&data_base);
+	if (status != KERN_OK) {
+		return status;
+	}
+
+	tracek("text_base=%08llx, data_base=%08llx", text_base, data_base);
+
+	*entry = text_base + bsp->bsp_trailer.bsp_exec_entry;
+	return KERN_OK;
+}
+
+kern_status_t bsp_launch_async(struct bsp *bsp, struct task *task)
+{
+	virt_addr_t stack_buffer, bsp_data_base;
+	virt_addr_t entry, sp;
+	kern_status_t status;
+
+	struct vm_object *user_stack = vm_object_create(
+		"stack",
+		5,
+		BOOTSTRAP_STACK_SIZE,
+		VM_PROT_READ | VM_PROT_WRITE | VM_PROT_USER);
+	if (!user_stack) {
+		return KERN_NO_ENTRY;
+	}
+
+	status = vm_region_map_object(
+		task->t_address_space,
+		VM_REGION_ANY_OFFSET,
+		user_stack,
+		0,
+		BOOTSTRAP_STACK_SIZE,
+		VM_PROT_READ | VM_PROT_WRITE | VM_PROT_USER,
+		&stack_buffer);
+
+	if (status != KERN_OK) {
+		return status;
+	}
+
+	status = vm_region_map_object(
+		task->t_address_space,
+		VM_REGION_ANY_OFFSET,
+		bsp->bsp_vmo,
+		0,
+		bsp->bsp_trailer.bsp_exec_offset,
+		VM_PROT_READ | VM_PROT_USER,
+		&bsp_data_base);
+
+	if (status != KERN_OK) {
+		return status;
+	}
+
+	status = map_executable(bsp, task, &entry);
+	if (status != KERN_OK) {
+		return status;
+	}
+#ifdef TRACE
+	vm_region_dump(task->t_address_space);
+#endif
+
+	sp = stack_buffer + BOOTSTRAP_STACK_SIZE;
+	tracek("bootstrap: entry=%llx, sp=%llx", entry, sp);
+
+	kern_handle_t self, self_address_space;
+	task_open_handle(task, &task->t_base, 0, &self);
+	task_open_handle(
+		task,
+		&task->t_address_space->vr_base,
+		0,
+		&self_address_space);
+
+	const uintptr_t args[] = {
+		0,                    // int argc
+		0,                    // const char ** argv
+		self,                 // kern_handle_t task
+		self_address_space,   // kern_handle_t address_space
+
+		/* this parameter is specific to the bsp bootstrap program, so
+	         * that it can access the rest of the bsp image. */
+		bsp_data_base,
+	};
+	const size_t nr_args = sizeof args / sizeof args[0];
+
+	struct thread *init_thread = task_create_thread(task);
+	thread_init_user(init_thread, entry, sp, args, nr_args);
+	schedule_thread_on_cpu(init_thread);
+
+	return KERN_OK;
+}

kernel/channel.c

@@ -0,0 +1,240 @@
+#include <kernel/channel.h>
+#include <kernel/msg.h>
+#include <kernel/util.h>
+#include <kernel/vm-region.h>
+
+#define CHANNEL_CAST(p) OBJECT_C_CAST(struct channel, c_base, &channel_type, p)
+
+static struct object_type channel_type = {
+	.ob_name = "channel",
+	.ob_size = sizeof(struct channel),
+	.ob_header_offset = offsetof(struct channel, c_base),
+};
+
+BTREE_DEFINE_SIMPLE_GET(struct kmsg, msgid_t, msg_node, msg_id, get_msg_with_id)
+
+kern_status_t channel_type_init(void)
+{
+	return object_type_register(&channel_type);
+}
+
+extern struct channel *channel_create(void)
+{
+	struct object *channel_object = object_create(&channel_type);
+	if (!channel_object) {
+		return NULL;
+	}
+
+	struct channel *channel = CHANNEL_CAST(channel_object);
+
+	return channel;
+}
+
+static bool try_enqueue(struct btree *tree, struct kmsg *msg)
+{
+	if (!tree->b_root) {
+		tree->b_root = &msg->msg_node;
+		btree_insert_fixup(tree, &msg->msg_node);
+		return true;
+	}
+
+	struct btree_node *cur = tree->b_root;
+	while (1) {
+		struct kmsg *cur_node
+			= BTREE_CONTAINER(struct kmsg, msg_node, cur);
+		struct btree_node *next = NULL;
+
+		if (msg->msg_id > cur_node->msg_id) {
+			next = btree_right(cur);
+
+			if (!next) {
+				btree_put_right(cur, &msg->msg_node);
+				break;
+			}
+		} else if (msg->msg_id < cur_node->msg_id) {
+			next = btree_left(cur);
+
+			if (!next) {
+				btree_put_left(cur, &msg->msg_node);
+				break;
+			}
+		} else {
+			return false;
+		}
+
+		cur = next;
+	}
+
+	btree_insert_fixup(tree, &msg->msg_node);
+	return true;
+}
+
+static void kmsg_reply_error(struct kmsg *msg, kern_status_t status)
+{
+	msg->msg_status = KMSG_REPLY_SENT;
+	msg->msg_status = status;
+	thread_awaken(msg->msg_sender_thread);
+}
+
+static struct kmsg *get_next_msg(struct channel *channel)
+{
+	unsigned long flags;
+	struct btree_node *cur = btree_first(&channel->c_msg);
+	while (cur) {
+		struct kmsg *msg = BTREE_CONTAINER(struct kmsg, msg_node, cur);
+		spin_lock_irqsave(&msg->msg_lock, &flags);
+		if (msg->msg_status == KMSG_WAIT_RECEIVE) {
+			msg->msg_status = KMSG_WAIT_REPLY;
+			return msg;
+		}
+
+		spin_unlock_irqrestore(&msg->msg_lock, flags);
+		cur = btree_next(cur);
+	}
+
+	return NULL;
+}
+
+extern kern_status_t channel_enqueue_msg(
+	struct channel *channel,
+	struct kmsg *msg)
+{
+	fill_random(&msg->msg_id, sizeof msg->msg_id);
+	while (!try_enqueue(&channel->c_msg, msg)) {
+		msg->msg_id++;
+	}
+
+	wakeup_one(&channel->c_wq);
+
+	return KERN_OK;
+}
+
+extern kern_status_t channel_recv_msg(
+	struct channel *channel,
+	struct msg *out_msg,
+	msgid_t *out_id,
+	unsigned long *irq_flags)
+{
+	struct wait_item waiter;
+	struct thread *self = current_thread();
+	struct kmsg *msg = NULL;
+
+	wait_item_init(&waiter, self);
+	for (;;) {
+		thread_wait_begin(&waiter, &channel->c_wq);
+		msg = get_next_msg(channel);
+		if (msg) {
+			break;
+		}
+
+		object_unlock_irqrestore(&channel->c_base, *irq_flags);
+		schedule(SCHED_NORMAL);
+		object_lock_irqsave(&channel->c_base, irq_flags);
+	}
+	thread_wait_end(&waiter, &channel->c_wq);
+
+	/* msg is now set to the next message to process */
+
+	struct task *sender = msg->msg_sender_thread->tr_parent;
+	struct task *receiver = self->tr_parent;
+
+	kern_status_t status = vm_region_memmove_v(
+		receiver->t_address_space,
+		0,
+		out_msg->msg_data,
+		out_msg->msg_data_count,
+		sender->t_address_space,
+		0,
+		msg->msg_req->msg_data,
+		msg->msg_req->msg_data_count,
+		VM_REGION_COPY_ALL);
+	if (status != KERN_OK) {
+		kmsg_reply_error(msg, status);
+		return status;
+	}
+
+	status = handle_list_transfer(
+		receiver->t_handles,
+		out_msg->msg_handles,
+		out_msg->msg_handles_count,
+		sender->t_handles,
+		msg->msg_req->msg_handles,
+		msg->msg_req->msg_handles_count);
+	if (status != KERN_OK) {
+		kmsg_reply_error(msg, status);
+		return status;
+	}
+
+	kmsg_reply_error(msg, KERN_OK);
+	return KERN_OK;
+}
+
+extern kern_status_t channel_reply_msg(
+	struct channel *channel,
+	msgid_t id,
+	const struct msg *resp,
+	unsigned long *irq_flags)
+{
+	struct kmsg *msg = get_msg_with_id(&channel->c_msg, id);
+	if (!msg || msg->msg_status != KMSG_WAIT_REPLY) {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	struct thread *self = current_thread();
+	struct task *sender = msg->msg_sender_thread->tr_parent;
+	struct task *receiver = self->tr_parent;
+
+	kern_status_t status = vm_region_memmove_v(
+		receiver->t_address_space,
+		0,
+		msg->msg_resp->msg_data,
+		msg->msg_resp->msg_data_count,
+		sender->t_address_space,
+		0,
+		resp->msg_data,
+		resp->msg_data_count,
+		VM_REGION_COPY_ALL);
+	if (status != KERN_OK) {
+		kmsg_reply_error(msg, status);
+		return status;
+	}
+
+	status = handle_list_transfer(
+		receiver->t_handles,
+		msg->msg_resp->msg_handles,
+		msg->msg_resp->msg_handles_count,
+		sender->t_handles,
+		resp->msg_handles,
+		resp->msg_handles_count);
+	if (status != KERN_OK) {
+		kmsg_reply_error(msg, status);
+		return status;
+	}
+
+	msg->msg_status = KERN_OK;
+	msg->msg_status = KMSG_REPLY_SENT;
+
+	return KERN_UNIMPLEMENTED;
+}
+
+extern kern_status_t channel_read_msg(
+	struct channel *channel,
+	msgid_t msg,
+	size_t offset,
+	void *buf,
+	size_t len,
+	size_t *nr_read)
+{
+	return KERN_UNIMPLEMENTED;
+}
+
+extern kern_status_t channel_write_msg(
+	struct channel *channel,
+	msgid_t msg,
+	size_t offset,
+	const void *buf,
+	size_t len,
+	size_t *nr_written)
+{
+	return KERN_UNIMPLEMENTED;
+}

kernel/clock.c

@@ -1,6 +1,6 @@
-#include <mango/clock.h>
-#include <mango/printk.h>
-#include <mango/compiler.h>
+#include <kernel/clock.h>
+#include <kernel/printk.h>
+#include <kernel/compiler.h>
 
 static clock_ticks_t ticks_per_sec = 0;
 volatile clock_ticks_t clock_ticks = 0;

kernel/console.c

@@ -1,7 +1,7 @@
-#include <mango/console.h>
-#include <mango/queue.h>
-#include <mango/locks.h>
-#include <mango/libc/string.h>
+#include <kernel/console.h>
+#include <kernel/queue.h>
+#include <kernel/locks.h>
+#include <kernel/libc/string.h>
 
 static struct queue consoles;
 static spin_lock_t consoles_lock = SPIN_LOCK_INIT;

kernel/cpu.c

@@ -1,6 +1,6 @@
-#include <mango/cpu.h>
-#include <mango/percpu.h>
-#include <mango/bitmap.h>
+#include <kernel/cpu.h>
+#include <kernel/percpu.h>
+#include <kernel/bitmap.h>
 
 DECLARE_BITMAP(cpu_available, CPU_MAX);
 DECLARE_BITMAP(cpu_online, CPU_MAX);

kernel/handle.c

@@ -0,0 +1,313 @@
+#include <kernel/handle.h>
+#include <kernel/libc/string.h>
+#include <kernel/object.h>
+#include <kernel/sched.h>
+#include <kernel/util.h>
+#include <kernel/vm.h>
+
+/* depth=3 gives a maximum of ~66.6 million handles */
+#define MAX_TABLE_DEPTH  3
+#define RESERVED_HANDLES 64
+
+static struct vm_cache handle_table_cache = {
+	.c_name = "handle_table",
+	.c_obj_size = sizeof(struct handle_table),
+};
+
+struct handle_table *handle_table_create(void)
+{
+	if (!VM_CACHE_INITIALISED(&handle_table_cache)) {
+		vm_cache_init(&handle_table_cache);
+	}
+
+	struct handle_table *out
+		= vm_cache_alloc(&handle_table_cache, VM_NORMAL);
+	if (!out) {
+		return NULL;
+	}
+
+	memset(out, 0x0, sizeof *out);
+
+	return out;
+}
+
+void handle_table_destroy(struct handle_table *tab)
+{
+}
+
+static kern_status_t decode_handle_indices(
+	kern_handle_t handle,
+	unsigned int indices[MAX_TABLE_DEPTH])
+{
+	for (int i = 0; i < MAX_TABLE_DEPTH; i++) {
+		unsigned int div = (i > 0 ? REFS_PER_TABLE : HANDLES_PER_TABLE);
+
+		unsigned int v = handle % div;
+		indices[MAX_TABLE_DEPTH - i - 1] = v;
+		handle /= div;
+	}
+
+	return handle == 0 ? KERN_OK : KERN_INVALID_ARGUMENT;
+}
+
+static kern_status_t encode_handle_indices(
+	unsigned int indices[MAX_TABLE_DEPTH],
+	kern_handle_t *out_handle)
+{
+	kern_handle_t handle = 0;
+	unsigned int mul = 1;
+
+	for (int i = MAX_TABLE_DEPTH - 1; i >= 0; i--) {
+		unsigned int v = indices[i] * mul;
+		handle += v;
+		mul *= REFS_PER_TABLE;
+	}
+
+	*out_handle = handle;
+	return KERN_OK;
+}
+
+kern_status_t handle_table_alloc_handle(
+	struct handle_table *tab,
+	struct handle **out_slot,
+	kern_handle_t *out_handle)
+{
+	int i;
+	unsigned int indices[MAX_TABLE_DEPTH] = {0};
+	static const unsigned int reserved_indices[MAX_TABLE_DEPTH] = {0};
+
+	for (i = 0; i < MAX_TABLE_DEPTH - 1; i++) {
+		unsigned int next_index = bitmap_lowest_clear(
+			tab->t_subtables.t_subtable_map,
+			REFS_PER_TABLE);
+
+		if (next_index == BITMAP_NPOS) {
+			return KERN_NO_ENTRY;
+		}
+
+		struct handle_table *next
+			= tab->t_subtables.t_subtable_list[next_index];
+		if (!next) {
+			next = handle_table_create();
+			tab->t_subtables.t_subtable_list[next_index] = next;
+		}
+
+		if (!next) {
+			return KERN_NO_MEMORY;
+		}
+
+		indices[i] = next_index;
+		tab = next;
+	}
+
+	if (memcmp(indices, reserved_indices, sizeof indices) == 0) {
+		bitmap_fill(tab->t_handles.t_handle_map, RESERVED_HANDLES);
+	}
+
+	unsigned int handle_index = bitmap_lowest_clear(
+		tab->t_handles.t_handle_map,
+		HANDLES_PER_TABLE);
+	if (handle_index == BITMAP_NPOS) {
+		return KERN_NO_ENTRY;
+	}
+
+	bitmap_set(tab->t_handles.t_handle_map, handle_index);
+	memset(&tab->t_handles.t_handle_list[handle_index],
+	       0x0,
+	       sizeof(struct handle));
+
+	indices[i] = handle_index;
+
+	*out_slot = &tab->t_handles.t_handle_list[handle_index];
+	return encode_handle_indices(indices, out_handle);
+}
+
+kern_status_t handle_table_free_handle(
+	struct handle_table *tab,
+	kern_handle_t handle)
+{
+	unsigned int indices[MAX_TABLE_DEPTH];
+	if (decode_handle_indices(handle, indices) != KERN_OK) {
+		return KERN_NO_ENTRY;
+	}
+
+	int i;
+	for (i = 0; i < MAX_TABLE_DEPTH - 1; i++) {
+		struct handle_table *next
+			= tab->t_subtables.t_subtable_list[indices[i]];
+		if (!next) {
+			return KERN_NO_ENTRY;
+		}
+
+		bitmap_clear(tab->t_subtables.t_subtable_map, indices[i]);
+		tab = next;
+	}
+
+	unsigned int handle_index = indices[i];
+	if (!bitmap_check(tab->t_handles.t_handle_map, handle_index)) {
+		return KERN_NO_ENTRY;
+	}
+
+	bitmap_clear(tab->t_handles.t_handle_map, handle_index);
+	struct handle *handle_entry
+		= &tab->t_handles.t_handle_list[handle_index];
+
+	if (handle_entry->h_object) {
+		object_remove_handle(handle_entry->h_object);
+	}
+
+	memset(handle_entry, 0x0, sizeof *handle_entry);
+	return KERN_OK;
+}
+
+struct handle *handle_table_get_handle(
+	struct handle_table *tab,
+	kern_handle_t handle)
+{
+	unsigned int indices[MAX_TABLE_DEPTH];
+	if (decode_handle_indices(handle, indices) != KERN_OK) {
+		return NULL;
+	}
+
+	int i;
+	for (i = 0; i < MAX_TABLE_DEPTH - 1; i++) {
+		struct handle_table *next
+			= tab->t_subtables.t_subtable_list[indices[i]];
+		if (!next) {
+			return NULL;
+		}
+
+		tab = next;
+	}
+
+	unsigned int handle_index = indices[i];
+	if (!bitmap_check(tab->t_handles.t_handle_map, handle_index)) {
+		return NULL;
+	}
+
+	if (!tab->t_handles.t_handle_list[handle_index].h_object) {
+		return NULL;
+	}
+
+	return &tab->t_handles.t_handle_list[handle_index];
+}
+
+struct handle_list_iterator {
+	struct handle_list *it_list;
+	size_t it_list_count;
+	size_t it_list_ptr;
+
+	kern_handle_t *it_handles;
+	size_t it_nr_handles;
+};
+
+static void handle_list_iterator_begin(
+	struct handle_list_iterator *it,
+	struct handle_list *list,
+	size_t list_count)
+{
+	memset(it, 0x0, sizeof *it);
+	it->it_list = list;
+	it->it_list_count = list_count;
+
+	while (it->it_list_ptr < list_count) {
+		if (list[it->it_list_ptr].l_nr_handles > 0) {
+			break;
+		}
+
+		it->it_list_ptr++;
+	}
+
+	if (it->it_list_ptr >= list_count) {
+		return;
+	}
+
+	it->it_handles = list[it->it_list_ptr].l_handles;
+	it->it_nr_handles = list[it->it_list_ptr].l_nr_handles;
+}
+
+static void handle_list_iterator_seek(
+	struct handle_list_iterator *it,
+	size_t nr_handles)
+{
+	if (nr_handles > it->it_nr_handles) {
+		nr_handles = it->it_nr_handles;
+	}
+
+	if (nr_handles < it->it_nr_handles) {
+		it->it_handles += nr_handles;
+		it->it_nr_handles -= nr_handles;
+		return;
+	}
+
+	it->it_list_ptr++;
+	while (it->it_list_ptr < it->it_list_count) {
+		if (it->it_list[it->it_list_ptr].l_nr_handles > 0) {
+			break;
+		}
+
+		it->it_list_ptr++;
+	}
+
+	if (it->it_list_ptr >= it->it_list_count) {
+		return;
+	}
+
+	it->it_handles = it->it_list[it->it_list_ptr].l_handles;
+	it->it_nr_handles = it->it_list[it->it_list_ptr].l_nr_handles;
+}
+
+kern_status_t handle_list_transfer(
+	struct handle_table *dest_table,
+	struct handle_list *dest_list,
+	size_t dest_list_count,
+	struct handle_table *src_table,
+	const struct handle_list *src_list,
+	size_t src_list_count)
+{
+	struct handle_list_iterator src, dest;
+	handle_list_iterator_begin(
+		&src,
+		(struct handle_list *)src_list,
+		src_list_count);
+	handle_list_iterator_begin(&dest, dest_list, dest_list_count);
+
+	while (src.it_nr_handles && dest.it_nr_handles) {
+		size_t to_copy = MIN(src.it_nr_handles, dest.it_nr_handles);
+		for (size_t i = 0; i < to_copy; i++) {
+			kern_handle_t handle_v = src.it_handles[i];
+			struct handle *handle
+				= handle_table_get_handle(src_table, handle_v);
+			if (!handle) {
+				return KERN_HANDLE_INVALID;
+			}
+
+			struct object *obj = object_ref(handle->h_object);
+			handle_flags_t flags = handle->h_flags;
+
+			handle_table_free_handle(src_table, handle_v);
+
+			struct handle *dest_slot = NULL;
+			kern_status_t status = handle_table_alloc_handle(
+				dest_table,
+				&dest_slot,
+				&handle_v);
+			if (status != KERN_OK) {
+				return status;
+			}
+
+			dest_slot->h_object = obj;
+			dest_slot->h_flags = flags;
+
+			object_add_handle(obj);
+			object_unref(obj);
+
+			dest.it_handles[i] = handle_v;
+		}
+
+		handle_list_iterator_seek(&src, to_copy);
+		handle_list_iterator_seek(&dest, to_copy);
+	}
+
+	return KERN_OK;
+}

kernel/iovec.c

@@ -0,0 +1,59 @@
+#include <kernel/iovec.h>
+#include <kernel/libc/string.h>
+#include <kernel/util.h>
+
+void iovec_iterator_begin(
+	struct iovec_iterator *it,
+	const struct iovec *vecs,
+	size_t nr_vecs)
+{
+	memset(it, 0x0, sizeof *it);
+	it->it_vecs = vecs;
+	it->it_nr_vecs = nr_vecs;
+
+	while (it->it_vec_ptr < nr_vecs) {
+		if (vecs[it->it_vec_ptr].io_len > 0) {
+			break;
+		}
+
+		it->it_vec_ptr++;
+	}
+
+	if (it->it_vec_ptr >= nr_vecs) {
+		return;
+	}
+
+	it->it_base = vecs[it->it_vec_ptr].io_base;
+	it->it_len = vecs[it->it_vec_ptr].io_len;
+}
+
+void iovec_iterator_seek(struct iovec_iterator *it, size_t nr_bytes)
+{
+	while (nr_bytes > 0) {
+		size_t to_seek = MIN(nr_bytes, it->it_len);
+
+		if (to_seek < it->it_len) {
+			it->it_len -= to_seek;
+			it->it_base += to_seek;
+			break;
+		}
+
+		nr_bytes -= to_seek;
+
+		it->it_vec_ptr++;
+		while (it->it_vec_ptr < it->it_nr_vecs) {
+			if (it->it_vecs[it->it_vec_ptr].io_len > 0) {
+				break;
+			}
+
+			it->it_vec_ptr++;
+		}
+
+		if (it->it_vec_ptr >= it->it_nr_vecs) {
+			return;
+		}
+
+		it->it_base = it->it_vecs[it->it_vec_ptr].io_base;
+		it->it_len = it->it_vecs[it->it_vec_ptr].io_len;
+	}
+}

kernel/object.c

@@ -1,6 +1,6 @@
-#include <mango/locks.h>
-#include <mango/object.h>
-#include <mango/queue.h>
+#include <kernel/locks.h>
+#include <kernel/object.h>
+#include <kernel/queue.h>
 
 #define HAS_OP(obj, opname) ((obj)->ob_type->ob_ops.opname)
 
@@ -21,7 +21,6 @@ kern_status_t object_type_register(struct object_type *p)
 
 	p->ob_cache.c_name = p->ob_name;
 	p->ob_cache.c_obj_size = p->ob_size;
-	p->ob_cache.c_page_order = VM_PAGE_16K;
 
 	vm_cache_init(&p->ob_cache);
 	p->ob_flags |= OBJTYPE_INIT;
@@ -71,7 +70,21 @@ struct object *object_ref(struct object *obj)
 	return obj;
 }
 
-void object_deref(struct object *obj)
+static void object_cleanup(struct object *obj, unsigned long flags)
+{
+	if (obj->ob_refcount > 0 || obj->ob_handles > 0) {
+		spin_unlock_irqrestore(&obj->ob_lock, flags);
+		return;
+	}
+
+	if (HAS_OP(obj, destroy)) {
+		obj->ob_type->ob_ops.destroy(obj);
+	}
+
+	vm_cache_free(&obj->ob_type->ob_cache, obj);
+}
+
+void object_unref(struct object *obj)
 {
 	unsigned long flags;
 	spin_lock_irqsave(&obj->ob_lock, &flags);
@@ -82,17 +95,26 @@ void object_deref(struct object *obj)
 	}
 
 	obj->ob_refcount--;
+	object_cleanup(obj, flags);
+}
 
-	if (obj->ob_refcount > 0) {
+void object_add_handle(struct object *obj)
+{
+	obj->ob_handles++;
+}
+
+void object_remove_handle(struct object *obj)
+{
+	unsigned long flags;
+	spin_lock_irqsave(&obj->ob_lock, &flags);
+
+	if (obj->ob_handles == 0) {
 		spin_unlock_irqrestore(&obj->ob_lock, flags);
 		return;
 	}
 
-	if (HAS_OP(obj, destroy)) {
-		obj->ob_type->ob_ops.destroy(obj);
-	}
-
-	vm_cache_free(&obj->ob_type->ob_cache, obj);
+	obj->ob_handles--;
+	object_cleanup(obj, flags);
 }
 
 void object_lock(struct object *obj)

kernel/panic.c

@@ -1,13 +1,13 @@
+#include <kernel/cpu.h>
+#include <kernel/libc/stdio.h>
+#include <kernel/machine/panic.h>
+#include <kernel/printk.h>
+#include <kernel/sched.h>
 #include <stdarg.h>
-#include <mango/machine/panic.h>
-#include <mango/libc/stdio.h>
-#include <mango/printk.h>
-#include <mango/sched.h>
-#include <mango/cpu.h>
 
 static int has_panicked = 0;
 
-void panic_irq(struct cpu_context *ctx, const char *fmt, ...)
+void panic_irq(struct ml_cpu_context *ctx, const char *fmt, ...)
 {
 	char buf[512];
 	va_list args;
@@ -22,7 +22,10 @@ void panic_irq(struct cpu_context *ctx, const char *fmt, ...)
 	struct thread *thr = current_thread();
 
 	if (task && thr) {
-		printk("task: %s (id: %d, thread: %d)", task->t_name, task->t_id, thr->tr_id);
+		printk("task: %s (id: %d, thread: %d)",
+		       task->t_name,
+		       task->t_id,
+		       thr->tr_id);
 	} else {
 		printk("task: [bootstrap]");
 	}

kernel/percpu.c

@@ -1,6 +1,6 @@
-#include <mango/percpu.h>
-#include <mango/cpu.h>
-#include <mango/vm.h>
+#include <kernel/percpu.h>
+#include <kernel/cpu.h>
+#include <kernel/vm.h>
 #include <stdint.h>
 #include <stddef.h>
 

kernel/port.c

@@ -0,0 +1,78 @@
+#include <kernel/channel.h>
+#include <kernel/port.h>
+#include <kernel/util.h>
+
+#define PORT_CAST(p) OBJECT_C_CAST(struct port, p_base, &port_type, p)
+
+static struct object_type port_type = {
+	.ob_name = "port",
+	.ob_size = sizeof(struct port),
+	.ob_header_offset = offsetof(struct port, p_base),
+};
+
+kern_status_t port_type_init(void)
+{
+	return object_type_register(&port_type);
+}
+
+struct port *port_cast(struct object *obj)
+{
+	return PORT_CAST(obj);
+}
+
+static void wait_for_reply(struct port *port)
+{
+}
+
+struct port *port_create(void)
+{
+	struct object *port_object = object_create(&port_type);
+	if (!port_object) {
+		return NULL;
+	}
+
+	struct port *port = PORT_CAST(port_object);
+
+	port->p_status = PORT_OFFLINE;
+
+	return port;
+}
+
+kern_status_t port_connect(struct port *port, struct channel *remote)
+{
+	if (port->p_status != PORT_OFFLINE) {
+		return KERN_BAD_STATE;
+	}
+
+	port->p_remote = remote;
+	port->p_status = PORT_READY;
+	return KERN_OK;
+}
+
+kern_status_t port_send_msg(
+	struct port *port,
+	const struct msg *req,
+	struct msg *resp)
+{
+	if (port->p_status != PORT_READY) {
+		return KERN_BAD_STATE;
+	}
+
+	struct thread *self = current_thread();
+	struct kmsg *msg = &self->tr_msg;
+	msg->msg_sender_thread = self;
+	msg->msg_sender_port = port;
+	msg->msg_req = req;
+	msg->msg_resp = resp;
+
+	unsigned long flags;
+	channel_lock_irqsave(port->p_remote, &flags);
+	channel_enqueue_msg(port->p_remote, msg);
+	channel_unlock_irqrestore(port->p_remote, flags);
+
+	port->p_status = PORT_SEND_BLOCKED;
+
+	wait_for_reply(port);
+
+	return msg->msg_result;
+}

kernel/printk.c

@@ -1,7 +1,7 @@
-#include <mango/printk.h>
-#include <mango/locks.h>
-#include <mango/console.h>
-#include <mango/libc/stdio.h>
+#include <kernel/printk.h>
+#include <kernel/locks.h>
+#include <kernel/console.h>
+#include <kernel/libc/stdio.h>
 #include <stdarg.h>
 
 #define LOG_BUFFER_SIZE 0x40000

libc/ctype/ctype.c

@@ -18,13 +18,18 @@
 #include <stdbool.h>
 #include <stdint.h>
 
-static unsigned int random_seed = 53455346;
+int isupper(int c)
+{
+	return (c >= 65 && c <= 90);
+}
 
-int isupper(int c) { return (c >= 65 && c <= 90); }
+int islower(int c)
+{
+	return (c >= 97 && c <= 122);
+}
 
-int islower(int c) { return (c >= 97 && c <= 122); }
-
-int toupper(int c) {
+int toupper(int c)
+{
 	if (!islower(c)) {
 		return c;
 	}
@@ -32,7 +37,8 @@ int toupper(int c) {
 	return c - 32;
 }
 
-int tolower(int c) {
+int tolower(int c)
+{
 	if (!isupper(c)) {
 		return c;
 	}
@@ -40,55 +46,48 @@ int tolower(int c) {
 	return c + 32;
 }
 
-int isdigit(int c) { return (c >= 48 && c <= 57); }
-
-int isalpha(int c) { return (c >= 65 && c <= 90) || (c >= 97 && c <= 122); }
-
-int isalnum(int c) { return isalpha(c) | isdigit(c); }
-
-int iscntrl(int c) { return (c <= 31) || (c == 127); }
-
-int isprint(int c) { return (c >= 32 && c <= 126) || (c >= 128 && c <= 254); }
-
-int isgraph(int c) { return isprint(c) && c != 32; }
-
-int ispunct(int c) { return isgraph(c) && !isalnum(c); }
-
-int isspace(int c) {
-    return (c == ' ') || (c == '\t') || (c == '\n') || (c == '\v') ||
-           (c == '\f') || (c == '\r');
+int isdigit(int c)
+{
+	return (c >= 48 && c <= 57);
 }
 
-int isxdigit(int c) {
+int isalpha(int c)
+{
+	return (c >= 65 && c <= 90) || (c >= 97 && c <= 122);
+}
+
+int isalnum(int c)
+{
+	return isalpha(c) | isdigit(c);
+}
+
+int iscntrl(int c)
+{
+	return (c <= 31) || (c == 127);
+}
+
+int isprint(int c)
+{
+	return (c >= 32 && c <= 126) || (c >= 128 && c <= 254);
+}
+
+int isgraph(int c)
+{
+	return isprint(c) && c != 32;
+}
+
+int ispunct(int c)
+{
+	return isgraph(c) && !isalnum(c);
+}
+
+int isspace(int c)
+{
+	return (c == ' ') || (c == '\t') || (c == '\n') || (c == '\v')
+	    || (c == '\f') || (c == '\r');
+}
+
+int isxdigit(int c)
+{
 	return isdigit(c) || (c >= 65 && c <= 70) || (c >= 97 && c <= 102);
 }
-
-bool fill_random(unsigned char *buffer, unsigned int size) {
-    if (!buffer || !size) {
-        return false;
-    }
-
-    for (uint32_t i = 0; i < size; i++) {
-        uint32_t next = random_seed;
-        uint32_t result;
-
-        next *= 1103515245;
-        next += 12345;
-        result = (uint32_t)(next / 65536) % 2048;
-
-        next *= 1103515245;
-        next += 12345;
-        result <<= 10;
-        result ^= (uint32_t)(next / 65536) % 1024;
-
-        next *= 1103515245;
-        next += 12345;
-        result <<= 10;
-        result ^= (uint32_t)(next / 65536) % 1024;
-        random_seed = next;
-
-        buffer[i] = (uint8_t)(result % 256);
-    }
-
-    return true;
-}