vm: implement demand-paging via userspace services with vm-controller

include/kernel/address-space.h

@@ -110,7 +110,9 @@ extern bool address_space_validate_access(
 
 /* find the mapping corresponding to the given virtual address, and page-in the
  * necessary vm_page to allow the memory access to succeed. if the relevant
- * vm-object page hasn't been allocated yet, it will be allocated here. */
+ * vm-object page hasn't been allocated yet, it will be allocated here.
+ * this function must be called with `region` UNLOCKED and interrupts ENABLED.
+ */
 extern kern_status_t address_space_demand_map(
 	struct address_space *region,
 	virt_addr_t addr,

include/kernel/syscall.h

@@ -170,6 +170,33 @@ extern kern_status_t sys_kern_object_wait(
 	kern_wait_item_t *items,
 	size_t nr_items);
 
+extern kern_status_t sys_vm_controller_create(kern_handle_t *out);
+extern kern_status_t sys_vm_controller_recv(
+	kern_handle_t ctrl,
+	equeue_packet_page_request_t *out);
+extern kern_status_t sys_vm_controller_recv_async(
+	kern_handle_t ctrl,
+	kern_handle_t eq,
+	equeue_key_t key);
+extern kern_status_t sys_vm_controller_create_object(
+	kern_handle_t ctrl,
+	const char *name,
+	size_t name_len,
+	equeue_key_t key,
+	size_t data_len,
+	vm_prot_t prot,
+	kern_handle_t *out);
+extern kern_status_t sys_vm_controller_detach_object(
+	kern_handle_t ctrl,
+	kern_handle_t vmo);
+extern kern_status_t sys_vm_controller_supply_pages(
+	kern_handle_t ctrl,
+	kern_handle_t dst_vmo,
+	off_t dst_offset,
+	kern_handle_t src_vmo,
+	off_t src_offset,
+	size_t count);
+
 extern virt_addr_t syscall_get_function(unsigned int sysid);
 
 #endif

include/kernel/vm-controller.h

@@ -18,16 +18,24 @@ enum page_request_status {
 struct vm_controller {
 	struct object vc_base;
 	/* tree of struct vm_objects bound to this controller, keyed with the
-	 * vm_key_t specified when the object(s) were created. */
+	 * equeue_key_t specified when the object(s) were created. */
 	struct btree vc_objects;
 	/* tree of pending page requests */
 	struct btree vc_requests;
+	/* the equeue to send async page requests to */
+	struct equeue *vc_eq;
+	equeue_key_t vc_eq_key;
+	/* the number of page requests queued with status PAGE_REQUEST_PENDING.
+	 * used to assert/clear VM_CONTROLLER_SIGNAL_REQUEST_RECEIVED */
+	size_t vc_requests_waiting;
 };
 
 struct page_request {
 	uint64_t req_id;
+	unsigned int req_type;
 	enum page_request_status req_status;
 	kern_status_t req_result;
+	spin_lock_t req_lock;
 	struct vm_object *req_object;
 	struct thread *req_sender;
 	struct btree_node req_node;
@@ -45,7 +53,8 @@ extern kern_status_t vm_controller_recv(
 	equeue_packet_page_request_t *out);
 extern kern_status_t vm_controller_recv_async(
 	struct vm_controller *ctrl,
-	struct equeue *eq);
+	struct equeue *eq,
+	equeue_key_t key);
 
 extern kern_status_t vm_controller_create_object(
 	struct vm_controller *ctrl,
@@ -58,10 +67,18 @@ extern kern_status_t vm_controller_create_object(
 extern kern_status_t vm_controller_detach_object(
 	struct vm_controller *ctrl,
 	struct vm_object *vmo);
+extern kern_status_t vm_controller_supply_pages(
+	struct vm_controller *ctrl,
+	struct vm_object *dst,
+	off_t dst_offset,
+	struct vm_object *src,
+	off_t src_offset,
+	size_t count);
 
 extern kern_status_t vm_controller_send_request(
 	struct vm_controller *ctrl,
-	struct page_request *req);
+	struct page_request *req,
+	unsigned long *irq_flags);
 
 DEFINE_OBJECT_LOCK_FUNCTION(vm_controller, vc_base)
 

include/kernel/vm-object.h

@@ -6,10 +6,27 @@
 
 #define VM_OBJECT_NAME_MAX 64
 
+struct vm_controller;
+
 enum vm_object_flags {
 	/* the memory behind this vm-object wasn't allocated by us, and
 	 * therefore shouldn't be freed by us */
 	VMO_IN_PLACE = 0x01u,
+	/* this vm-object is/was attached to a vm-controller */
+	VMO_CONTROLLER = 0x02u,
+
+	/* these flags are for use with vm_object_get_page */
+	/**************************************************/
+
+	/* if the relevant page hasn't been allocated yet, it will be allocated
+	 * and returned. if this flag isn't specified, NULL will be returned. */
+	VMO_ALLOCATE_MISSING_PAGE = 0x04u,
+	/* if the vm-object is attached to a vm-controller, and the relevant
+	 * page is uncommitted, send a request to the vm-controller to provide
+	 * the missing page. will result in the vm-object being unlocked and
+	 * the current thread sleeping until the request is fulfilled. the
+	 * vm-object will be re-locked before the function returns. */
+	VMO_REQUEST_MISSING_PAGE = 0x08u,
 };
 
 struct vm_object {
@@ -21,8 +38,12 @@ struct vm_object {
 	/* queue of struct vm_region_mapping */
 	struct queue vo_mappings;
 
-	/* memory protection flags. mappings of this vm_object can only use
-	 * a subset of the flags set in this mask. */
+	struct vm_controller *vo_ctrl;
+	equeue_key_t vo_key;
+	struct btree_node vo_ctrl_node;
+
+	/* memory protection flags. mappings of this vm_object can only
+	 * use a subset of the flags set in this mask. */
 	vm_prot_t vo_prot;
 
 	/* btree of vm_pages that have been allocated to this vm_object.
@@ -58,13 +79,10 @@ extern struct vm_object *vm_object_create_in_place(
 	vm_prot_t prot);
 
 extern struct vm_page *vm_object_get_page(
-	const struct vm_object *vo,
-	off_t offset);
-
-extern struct vm_page *vm_object_alloc_page(
 	struct vm_object *vo,
 	off_t offset,
-	enum vm_page_order size);
+	enum vm_object_flags flags,
+	unsigned long *irq_flags);
 
 extern kern_status_t vm_object_read(
 	struct vm_object *vo,