kernel: implement cpu IDs and per-cpu variables

arch/x86_64/acpi/acpi.c

@@ -3,10 +3,5 @@
 
 kern_status_t acpi_init(void)
 {
-	kern_status_t status = acpi_smp_init();
-	if (status != KERN_OK) {
-		return status;
-	}
-	
 	return KERN_OK;
 }

arch/x86_64/acpi/smp.c

@@ -1,6 +1,7 @@
 #include <arch/acpi.h>
 #include <socks/printk.h>
 #include <socks/vm.h>
+#include <socks/cpu.h>
 
 #define AP_TRAMPOLINE_PADDR   0x8000
 #define LAPIC_IPI_STATUS_REG  0x0280
@@ -26,7 +27,7 @@ struct lapic_override_record {
 extern uint8_t acpi_bsp_lapic_id(void);
 extern char ap_trampoline[];
 
-static int send_ipi(void *lapic, unsigned int target_id, uint32_t payload)
+static int __used send_ipi(void *lapic, unsigned int target_id, uint32_t payload)
 {
 	uintptr_t lapic_ptr = (uintptr_t)lapic;
 
@@ -38,7 +39,7 @@ static int send_ipi(void *lapic, unsigned int target_id, uint32_t payload)
 	return 0;
 }
 
-static int init_ap(struct acpi_madt_record *rec, void *bsp_lapic, uint8_t bsp_id)
+static int __used init_ap(struct acpi_madt_record *rec, void *bsp_lapic, uint8_t bsp_id)
 {
 	struct lapic_record *lapic = (struct lapic_record *)(rec + 1);
 	if (!(lapic->l_flags & 0x1) && !(lapic->l_flags & 0x2)) {
@@ -60,7 +61,7 @@ static int init_ap(struct acpi_madt_record *rec, void *bsp_lapic, uint8_t bsp_id
 	return 0;
 }
 
-static void *find_lapic(struct acpi_madt *madt)
+static void *__used find_lapic(struct acpi_madt *madt)
 {
 	phys_addr_t local_apic = madt->m_lapic_ptr;
 
@@ -85,6 +86,7 @@ static void *find_lapic(struct acpi_madt *madt)
 	return vm_phys_to_virt(local_apic);
 }
 
+/*
 kern_status_t acpi_smp_init(void)
 {
 	struct acpi_madt *madt = (struct acpi_madt *)acpi_find_sdt(ACPI_SIG_MADT);
@@ -118,6 +120,54 @@ kern_status_t acpi_smp_init(void)
 		p += rec->r_length;
 	}
 
-	printk("acpi: initialised %u processors", nr_processors);
+	printk("acpi: found %u logical cores", nr_processors);
+	return KERN_OK;
+}
+*/
+
+kern_status_t acpi_scan_cpu_topology(void)
+{
+	struct acpi_madt *madt = (struct acpi_madt *)acpi_find_sdt(ACPI_SIG_MADT);
+	if (!madt) {
+		return KERN_UNSUPPORTED;
+	}
+
+	uint8_t bsp_id = acpi_bsp_lapic_id();
+
+	//void *bsp_lapic = find_lapic(madt);
+
+	unsigned char *p = (unsigned char *)madt + sizeof *madt;
+	unsigned char *madt_end = (unsigned char *)madt + madt->m_base.s_length;
+
+	unsigned int nr_processors = 0;
+
+	while (p < madt_end) {
+		struct acpi_madt_record *rec = (struct acpi_madt_record *)p;
+
+		if (rec->r_type != ACPI_MADT_LAPIC) {
+			p += rec->r_length;
+			continue;
+		}
+
+		struct lapic_record *lapic = (struct lapic_record *)(rec + 1);
+
+		if (!(lapic->l_flags & 0x1) && !(lapic->l_flags & 0x2)) {
+			/* processor cannot be used */
+			continue;
+		}
+
+		cpu_set_available(lapic->l_apic_id);
+		nr_processors++;
+
+		p += rec->r_length;
+	}
+
+	cpu_set_online(bsp_id);
+
+	/* store the BSP ID in the current cpu block */
+	ml_cpu_block *self = ml_this_cpu();
+	self->c_cpu_id = bsp_id;
+
+	printk("acpi: found %u logical cores", nr_processors);
 	return KERN_OK;
 }

arch/x86_64/cpu.c

@@ -1,7 +1,9 @@
 #include <socks/machine/cpu.h>
+#include <arch/msr.h>
 
 int ml_cpu_block_init(ml_cpu_block *p)
 {
+	p->c_this = p;
 	gdt_init(&p->c_gdt, &p->c_gdt_ptr);
 	idt_init(&p->c_idt_ptr);
 	return 0;
@@ -11,5 +13,6 @@ int ml_cpu_block_use(ml_cpu_block *p)
 {
 	gdt_load(&p->c_gdt_ptr);
 	idt_load(&p->c_idt_ptr);
+	wrmsr(MSR_GS_BASE, (uint64_t)p);
 	return 0;
 }

arch/x86_64/cpu_ctrl.S

@@ -6,3 +6,35 @@ ml_halt_cpu:
 1:	cli
 	hlt
 	jmp 1b
+
+
+	.global ml_this_cpu
+	.type ml_this_cpu, @function
+
+ml_this_cpu:
+	movq %gs:0, %rax
+	ret
+
+
+	.global rdmsr
+	.type rdmsr, @function
+
+rdmsr:
+	mov %rdi, %rcx
+	rdmsr
+	shl $32, %rdx
+	orq %rax, %rdx
+	ret
+
+	.global wrmsr
+	.type wrmsr, @function
+
+wrmsr:
+	mov %rdi, %rcx
+	mov %rsi, %rax
+	mov %rsi, %rdx
+	shr $32, %rdx
+
+	wrmsr
+
+	ret

arch/x86_64/include/arch/acpi.h

@@ -66,7 +66,7 @@ struct acpi_madt {
 } __packed;
 
 extern kern_status_t acpi_init(void);
-extern kern_status_t acpi_smp_init(void);
+extern kern_status_t acpi_scan_cpu_topology(void);
 
 extern struct acpi_sdt *acpi_find_sdt(uint32_t sig);
 

arch/x86_64/include/arch/msr.h

@@ -0,0 +1,12 @@
+#ifndef ARCH_MSR_H_
+#define ARCH_MSR_H_
+
+#include <stdint.h>
+
+#define MSR_GS_BASE 0xC0000101
+
+/* defined in cpu_ctrl.S */
+extern uint64_t rdmsr(uint32_t id);
+extern void wrmsr(uint32_t id, uint64_t val);
+
+#endif

arch/x86_64/include/socks/machine/cpu.h

@@ -4,11 +4,16 @@
 #include <arch/gdt.h>
 #include <arch/irq.h>
 
+#define ml_cpu_block_get_id(p) ((p)->c_cpu_id)
+
 typedef struct ml_cpu_block {
+	struct ml_cpu_block *c_this;
+
 	struct gdt c_gdt;
 	struct gdt_ptr c_gdt_ptr;
 
 	struct idt_ptr c_idt_ptr;
+	unsigned int c_cpu_id;
 } ml_cpu_block;
 
 extern int ml_init_bootcpu(void);
@@ -18,5 +23,6 @@ extern int ml_cpu_block_use(ml_cpu_block *p);
 
 /* defined in cpu_ctrl.S */
 extern void ml_halt_cpu(void);
+extern ml_cpu_block *ml_this_cpu(void);
 
 #endif

arch/x86_64/init.c

@@ -3,6 +3,8 @@
 #include <socks/object.h>
 #include <arch/e820.h>
 #include <socks/init.h>
+#include <socks/percpu.h>
+#include <socks/cpu.h>
 #include <socks/memblock.h>
 #include <socks/vm.h>
 #include <socks/printk.h>
@@ -53,7 +55,13 @@ int ml_init(uintptr_t arg)
 
 	pmap_bootstrap();
 
-	acpi_init();
+	acpi_scan_cpu_topology();
+	init_per_cpu_areas();
+
+	cpu_data_t *this_cpu = get_this_cpu();
+	this_cpu->c_flags = CPU_ONLINE;
+	this_cpu->c_id = this_cpu();
+	put_cpu(this_cpu);
 
 	vm_zone_descriptor_t vm_zones[] = {
 		{ .zd_id = VM_ZONE_DMA, .zd_node = 0, .zd_name = "dma", .zd_base = 0x00, .zd_limit = 0xffffff },

include/socks/cpu.h

@@ -1,13 +1,33 @@
 #ifndef SOCKS_CPU_H_
 #define SOCKS_CPU_H_
 
-#include <socks/bitmap.h>
+#include <socks/machine/cpu.h>
+#include <socks/sched.h>
+
+typedef enum cpu_flags {
+	CPU_ONLINE = 0x01u,
+} cpu_flags_t;
+
+typedef struct cpu_data {
+	cpu_flags_t c_flags;
+	unsigned int c_id;
+
+	task_t *c_current_task;
+	runqueue_t c_rq;
+} cpu_data_t;
 
 /* maximum number of processor cores that the kernel can support.
    TODO move to build config option */
 #define CPU_MAX 128
 
-DECLARE_BITMAP(cpu_available, CPU_MAX);
+#define this_cpu() (ml_cpu_block_get_id(ml_this_cpu()))
-DECLARE_BITMAP(cpu_online, CPU_MAX);
+
+extern cpu_data_t *get_this_cpu(void);
+extern void put_cpu(cpu_data_t *cpu);
+
+extern void cpu_set_available(unsigned int cpu_id);
+extern void cpu_set_online(unsigned int cpu_id);
+
+extern unsigned int cpu_get_highest_available(void);
 
 #endif

include/socks/percpu.h

@@ -1,6 +1,7 @@
 #ifndef SOCKS_PERCPU_H_
 #define SOCKS_PERCPU_H_
 
+#include <socks/status.h>
 #include <socks/compiler.h>
 
 #define DEFINE_PERCPU_VAR(type, name) \
@@ -10,6 +11,7 @@
 
 #define percpu_put(var)
 
+extern kern_status_t init_per_cpu_areas(void);
 extern void *__percpu_get(void *var);
 
 #endif

include/socks/sched.h

@@ -64,10 +64,13 @@ typedef struct runqueue {
 
 extern kern_status_t sched_init(void);
 
+extern void runqueue_init(runqueue_t *rq);
+
 extern task_t *task_alloc(void);
 static inline task_t *task_ref(task_t *task) { return object_data(object_ref(object_header(task))); }
 static inline void task_deref(task_t *task) { object_deref(object_header(task)); }
 extern task_t *task_from_pid(unsigned int pid);
+extern task_t *kernel_task(void);
 
 static inline void task_lock_irqsave(task_t *task, unsigned long *flags)
 {

init/main.c

@@ -5,7 +5,7 @@
 #include <socks/object.h>
 #include <socks/sched.h>
 #include <socks/machine/init.h>
-#include <socks/machine/cpu.h>
+#include <socks/cpu.h>
 
 extern unsigned long get_rflags(void);
 
@@ -23,6 +23,8 @@ void kernel_init(uintptr_t arg)
 	object_bootstrap();
 	sched_init();
 
+	printk("kernel_init() running on processor %u", this_cpu());
+
 	run_all_tests();
 
 	ml_halt_cpu();

kernel/cpu.c

@@ -0,0 +1,41 @@
+#include <socks/cpu.h>
+#include <socks/percpu.h>
+#include <socks/bitmap.h>
+
+DECLARE_BITMAP(cpu_available, CPU_MAX);
+DECLARE_BITMAP(cpu_online, CPU_MAX);
+
+DEFINE_PERCPU_VAR(cpu_data_t, cpu_data);
+
+cpu_data_t *get_this_cpu(void)
+{
+	return percpu_get(&cpu_data);
+}
+
+void put_cpu(cpu_data_t *cpu)
+{
+	percpu_put(cpu);
+}
+
+void cpu_set_available(unsigned int cpu_id)
+{
+	if (cpu_id >= CPU_MAX) {
+		return;
+	}
+
+	bitmap_set(cpu_available, cpu_id);
+}
+
+void cpu_set_online(unsigned int cpu_id)
+{
+	if (cpu_id >= CPU_MAX) {
+		return;
+	}
+
+	bitmap_set(cpu_online, cpu_id);
+}
+
+unsigned int cpu_get_highest_available(void)
+{
+	return bitmap_highest_set(cpu_available, CPU_MAX);
+}

kernel/percpu.c

@@ -0,0 +1,40 @@
+#include <socks/percpu.h>
+#include <socks/cpu.h>
+#include <socks/vm.h>
+#include <stdint.h>
+#include <stddef.h>
+
+extern char __percpu_start[];
+extern char __percpu_end[];
+
+static void *percpu_buffer = NULL;
+static size_t percpu_stride = 0;
+
+extern kern_status_t init_per_cpu_areas(void)
+{
+	unsigned int last_cpu = cpu_get_highest_available();
+
+	percpu_stride = (uintptr_t)__percpu_end - (uintptr_t)__percpu_start;
+	if (percpu_stride & 0x7) {
+		percpu_stride &= ~0x7;
+		percpu_stride += 0x8;
+	}
+
+	percpu_buffer = kmalloc(last_cpu * percpu_stride, 0);
+	return KERN_OK;
+}
+
+extern void *__percpu_get(void *var)
+{
+	uintptr_t pvar = (uintptr_t)var;
+	uintptr_t percpu_start = (uintptr_t)__percpu_start;
+	uintptr_t percpu_end = (uintptr_t)__percpu_end;
+
+	if (pvar < percpu_start || pvar >= percpu_end) {
+		return NULL;
+	}
+
+	size_t var_offset = pvar - percpu_start;
+
+	return (char *)percpu_buffer + (this_cpu() * percpu_stride) + var_offset;
+}