[Qemu-devel] [PULL 37/57] postcopy: ram_enable_notify to switch on userfault

[Juan Quintela]

From: "Dr. David Alan Gilbert" 

Mark the area of RAM as 'userfault'
Start up a fault-thread to handle any userfaults we might receive
from it (to be filled in later)

Signed-off-by: Dr. David Alan Gilbert 
Reviewed-by: David Gibson 
Reviewed-by: Juan Quintela 
Reviewed-by: Amit Shah 
Signed-off-by: Juan Quintela 
---
 include/migration/migration.h|  3 ++
 include/migration/postcopy-ram.h |  6 
 migration/postcopy-ram.c | 69 
 migration/savevm.c   |  9 ++
 4 files changed, 87 insertions(+)

diff --git a/include/migration/migration.h b/include/migration/migration.h
index 6e42b58..2ad0d2b 100644
--- a/include/migration/migration.h
+++ b/include/migration/migration.h
@@ -86,6 +86,9 @@ struct MigrationIncomingState {
  */
 QemuEvent main_thread_load_event;

+QemuThread fault_thread;
+QemuSemaphore  fault_thread_sem;
+
 /* For the kernel to send us notifications */
 int   userfault_fd;
 QEMUFile *to_src_file;
diff --git a/include/migration/postcopy-ram.h b/include/migration/postcopy-ram.h
index f87020c..b10c03d 100644
--- a/include/migration/postcopy-ram.h
+++ b/include/migration/postcopy-ram.h
@@ -17,6 +17,12 @@
 bool postcopy_ram_supported_by_host(void);

 /*
+ * Make all of RAM sensitive to accesses to areas that haven't yet been written
+ * and wire up anything necessary to deal with it.
+ */
+int postcopy_ram_enable_notify(MigrationIncomingState *mis);
+
+/*
  * Initialise postcopy-ram, setting the RAM to a state where we can go into
  * postcopy later; must be called prior to any precopy.
  * called from ram.c's similarly named ram_postcopy_incoming_init
diff --git a/migration/postcopy-ram.c b/migration/postcopy-ram.c
index 8478bfd..3110b2a 100644
--- a/migration/postcopy-ram.c
+++ b/migration/postcopy-ram.c
@@ -275,6 +275,69 @@ int postcopy_ram_incoming_cleanup(MigrationIncomingState 
*mis)
 return 0;
 }

+/*
+ * Mark the given area of RAM as requiring notification to unwritten areas
+ * Used as a  callback on qemu_ram_foreach_block.
+ *   host_addr: Base of area to mark
+ *   offset: Offset in the whole ram arena
+ *   length: Length of the section
+ *   opaque: MigrationIncomingState pointer
+ * Returns 0 on success
+ */
+static int ram_block_enable_notify(const char *block_name, void *host_addr,
+   ram_addr_t offset, ram_addr_t length,
+   void *opaque)
+{
+MigrationIncomingState *mis = opaque;
+struct uffdio_register reg_struct;
+
+reg_struct.range.start = (uintptr_t)host_addr;
+reg_struct.range.len = length;
+reg_struct.mode = UFFDIO_REGISTER_MODE_MISSING;
+
+/* Now tell our userfault_fd that it's responsible for this area */
+if (ioctl(mis->userfault_fd, UFFDIO_REGISTER, _struct)) {
+error_report("%s userfault register: %s", __func__, strerror(errno));
+return -1;
+}
+
+return 0;
+}
+
+/*
+ * Handle faults detected by the USERFAULT markings
+ */
+static void *postcopy_ram_fault_thread(void *opaque)
+{
+MigrationIncomingState *mis = opaque;
+
+fprintf(stderr, "postcopy_ram_fault_thread\n");
+/* TODO: In later patch */
+qemu_sem_post(>fault_thread_sem);
+while (1) {
+/* TODO: In later patch */
+}
+
+return NULL;
+}
+
+int postcopy_ram_enable_notify(MigrationIncomingState *mis)
+{
+/* Create the fault handler thread and wait for it to be ready */
+qemu_sem_init(>fault_thread_sem, 0);
+qemu_thread_create(>fault_thread, "postcopy/fault",
+   postcopy_ram_fault_thread, mis, QEMU_THREAD_JOINABLE);
+qemu_sem_wait(>fault_thread_sem);
+qemu_sem_destroy(>fault_thread_sem);
+
+/* Mark so that we get notified of accesses to unwritten areas */
+if (qemu_ram_foreach_block(ram_block_enable_notify, mis)) {
+return -1;
+}
+
+return 0;
+}
+
 #else
 /* No target OS support, stubs just fail */
 bool postcopy_ram_supported_by_host(void)
@@ -301,6 +364,12 @@ int postcopy_ram_discard_range(MigrationIncomingState 
*mis, uint8_t *start,
 assert(0);
 return -1;
 }
+
+int postcopy_ram_enable_notify(MigrationIncomingState *mis)
+{
+assert(0);
+return -1;
+}
 #endif

 /* - */
diff --git a/migration/savevm.c b/migration/savevm.c
index 674f0fb..a7210a2 100644
--- a/migration/savevm.c
+++ b/migration/savevm.c
@@ -1381,6 +1381,15 @@ static int 
loadvm_postcopy_handle_listen(MigrationIncomingState *mis)
 return -1;
 }

+/*
+ * Sensitise RAM - can now generate requests for blocks that don't exist
+ * However, at this point the CPU shouldn't be running, and the IO
+ * shouldn't be doing anything yet so don't actually expect requests
+ */
+if 

[Qemu-devel] [PULL 37/57] postcopy: ram_enable_notify to switch on userfault

[Juan Quintela]

From: "Dr. David Alan Gilbert" 

Mark the area of RAM as 'userfault'
Start up a fault-thread to handle any userfaults we might receive
from it (to be filled in later)

Signed-off-by: Dr. David Alan Gilbert 
Reviewed-by: David Gibson 
Reviewed-by: Juan Quintela 
Reviewed-by: Amit Shah 
Signed-off-by: Juan Quintela 
---
 include/migration/migration.h|  3 ++
 include/migration/postcopy-ram.h |  6 
 migration/postcopy-ram.c | 69 
 migration/savevm.c   |  9 ++
 4 files changed, 87 insertions(+)

diff --git a/include/migration/migration.h b/include/migration/migration.h
index 6e42b58..2ad0d2b 100644
--- a/include/migration/migration.h
+++ b/include/migration/migration.h
@@ -86,6 +86,9 @@ struct MigrationIncomingState {
  */
 QemuEvent main_thread_load_event;

+QemuThread fault_thread;
+QemuSemaphore  fault_thread_sem;
+
 /* For the kernel to send us notifications */
 int   userfault_fd;
 QEMUFile *to_src_file;
diff --git a/include/migration/postcopy-ram.h b/include/migration/postcopy-ram.h
index f87020c..b10c03d 100644
--- a/include/migration/postcopy-ram.h
+++ b/include/migration/postcopy-ram.h
@@ -17,6 +17,12 @@
 bool postcopy_ram_supported_by_host(void);

 /*
+ * Make all of RAM sensitive to accesses to areas that haven't yet been written
+ * and wire up anything necessary to deal with it.
+ */
+int postcopy_ram_enable_notify(MigrationIncomingState *mis);
+
+/*
  * Initialise postcopy-ram, setting the RAM to a state where we can go into
  * postcopy later; must be called prior to any precopy.
  * called from ram.c's similarly named ram_postcopy_incoming_init
diff --git a/migration/postcopy-ram.c b/migration/postcopy-ram.c
index 8478bfd..3110b2a 100644
--- a/migration/postcopy-ram.c
+++ b/migration/postcopy-ram.c
@@ -275,6 +275,69 @@ int postcopy_ram_incoming_cleanup(MigrationIncomingState 
*mis)
 return 0;
 }

+/*
+ * Mark the given area of RAM as requiring notification to unwritten areas
+ * Used as a  callback on qemu_ram_foreach_block.
+ *   host_addr: Base of area to mark
+ *   offset: Offset in the whole ram arena
+ *   length: Length of the section
+ *   opaque: MigrationIncomingState pointer
+ * Returns 0 on success
+ */
+static int ram_block_enable_notify(const char *block_name, void *host_addr,
+   ram_addr_t offset, ram_addr_t length,
+   void *opaque)
+{
+MigrationIncomingState *mis = opaque;
+struct uffdio_register reg_struct;
+
+reg_struct.range.start = (uintptr_t)host_addr;
+reg_struct.range.len = length;
+reg_struct.mode = UFFDIO_REGISTER_MODE_MISSING;
+
+/* Now tell our userfault_fd that it's responsible for this area */
+if (ioctl(mis->userfault_fd, UFFDIO_REGISTER, _struct)) {
+error_report("%s userfault register: %s", __func__, strerror(errno));
+return -1;
+}
+
+return 0;
+}
+
+/*
+ * Handle faults detected by the USERFAULT markings
+ */
+static void *postcopy_ram_fault_thread(void *opaque)
+{
+MigrationIncomingState *mis = opaque;
+
+fprintf(stderr, "postcopy_ram_fault_thread\n");
+/* TODO: In later patch */
+qemu_sem_post(>fault_thread_sem);
+while (1) {
+/* TODO: In later patch */
+}
+
+return NULL;
+}
+
+int postcopy_ram_enable_notify(MigrationIncomingState *mis)
+{
+/* Create the fault handler thread and wait for it to be ready */
+qemu_sem_init(>fault_thread_sem, 0);
+qemu_thread_create(>fault_thread, "postcopy/fault",
+   postcopy_ram_fault_thread, mis, QEMU_THREAD_JOINABLE);
+qemu_sem_wait(>fault_thread_sem);
+qemu_sem_destroy(>fault_thread_sem);
+
+/* Mark so that we get notified of accesses to unwritten areas */
+if (qemu_ram_foreach_block(ram_block_enable_notify, mis)) {
+return -1;
+}
+
+return 0;
+}
+
 #else
 /* No target OS support, stubs just fail */
 bool postcopy_ram_supported_by_host(void)
@@ -301,6 +364,12 @@ int postcopy_ram_discard_range(MigrationIncomingState 
*mis, uint8_t *start,
 assert(0);
 return -1;
 }
+
+int postcopy_ram_enable_notify(MigrationIncomingState *mis)
+{
+assert(0);
+return -1;
+}
 #endif

 /* - */
diff --git a/migration/savevm.c b/migration/savevm.c
index 8cf4c53..9fd276e 100644
--- a/migration/savevm.c
+++ b/migration/savevm.c
@@ -1381,6 +1381,15 @@ static int 
loadvm_postcopy_handle_listen(MigrationIncomingState *mis)
 return -1;
 }

+/*
+ * Sensitise RAM - can now generate requests for blocks that don't exist
+ * However, at this point the CPU shouldn't be running, and the IO
+ * shouldn't be doing anything yet so don't actually expect requests
+ */
+if