Re: [PATCH v2 36/41] postcopy: implement incoming part of postcopy live migration

[Juan Quintela]

Isaku Yamahata  wrote:
> This patch implements postcopy live migration for incoming part
>
> Signed-off-by: Isaku Yamahata 


> +void ram_save_set_params(const MigrationParams *params, void *opaque);

> -register_savevm_live(NULL, "ram", 0, RAM_SAVE_VERSION_ID, NULL,
> - ram_save_live, NULL, ram_load, NULL);
> +register_savevm_live(NULL, "ram", 0, RAM_SAVE_VERSION_ID,
> + ram_save_set_params, ram_save_live, NULL,
> + incoming_postcopy ?
> + postcopy_incoming_ram_load : ram_load, NULL);


ram_save_set_params() used on this patch but defined on next one.

Later, Juan.
--
To unsubscribe from this list: send the line "unsubscribe kvm" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v2 36/41] postcopy: implement incoming part of postcopy live migration

[Juan Quintela]

Isaku Yamahata  wrote:
> This patch implements postcopy live migration for incoming part
>
>  vl.c   |8 +-
>  13 files changed, 1409 insertions(+), 21 deletions(-)
>  copy linux-headers/linux/umem.h => migration-postcopy-stub.c (55%)

Ouch, git got really confused.


> +void postcopy_incoming_prepare(void)
> +{
> +RAMBlock *block;
> +
> +if (!incoming_postcopy) {
> +return;
> +}

We are testing the negation of this before calling the function, it is
not needed in one of the sides?

> +
> +state.state = 0;
> +state.host_page_size = getpagesize();
> +state.host_page_shift = ffs(state.host_page_size) - 1;
> +state.version_id = RAM_SAVE_VERSION_ID; /* = save version of
> +   ram_save_live() */
> +
> +QLIST_FOREACH(block, &ram_list.blocks, next) {
> +block->umem = umem_new(block->host, block->length);
> +block->flags |= RAM_POSTCOPY_UMEM_MASK;
> +}
> +}
> +
> +static int postcopy_incoming_ram_load_get64(QEMUFile *f,
> +ram_addr_t *addr, int *flags)
> +{
> +*addr = qemu_get_be64(f);
> +*flags = *addr & ~TARGET_PAGE_MASK;
> +*addr &= TARGET_PAGE_MASK;
> +return qemu_file_get_error(f);
> +}
> +
> +int postcopy_incoming_ram_load(QEMUFile *f, void *opaque, int version_id)
> +{
> +ram_addr_t addr;
> +int flags;
> +int error;
> +
> +DPRINTF("incoming ram load\n");
> +/*
> + * RAM_SAVE_FLAGS_EOS or
> + * RAM_SAVE_FLAGS_MEM_SIZE + mem size + RAM_SAVE_FLAGS_EOS
> + * see postcopy_outgoing_ram_save_live()
> + */
> +
> +if (version_id != RAM_SAVE_VERSION_ID) {
> +DPRINTF("RAM_SAVE_VERSION_ID %d != %d\n",
> +version_id, RAM_SAVE_VERSION_ID);
> +return -EINVAL;
> +}
> +error = postcopy_incoming_ram_load_get64(f, &addr, &flags);
> +DPRINTF("addr 0x%lx flags 0x%x\n", addr, flags);
> +if (error) {
> +DPRINTF("error %d\n", error);
> +return error;
> +}
> +if (flags == RAM_SAVE_FLAG_EOS && addr == 0) {
> +DPRINTF("EOS\n");
> +return 0;
> +}
> +
> +if (flags != RAM_SAVE_FLAG_MEM_SIZE) {
> +DPRINTF("-EINVAL flags 0x%x\n", flags);
> +return -EINVAL;
> +}
> +error = ram_load_mem_size(f, addr);
> +if (error) {
> +DPRINTF("addr 0x%lx error %d\n", addr, error);
> +return error;
> +}
> +
> +error = postcopy_incoming_ram_load_get64(f, &addr, &flags);
> +if (error) {
> +DPRINTF("addr 0x%lx flags 0x%x error %d\n", addr, flags, error);
> +return error;
> +}
> +if (flags == RAM_SAVE_FLAG_EOS && addr == 0) {
> +DPRINTF("done\n");
> +return 0;
> +}
> +DPRINTF("-EINVAL\n");
> +return -EINVAL;
> +}
> +
> +static void postcopy_incoming_pipe_and_fork_umemd(int mig_read_fd,
> +  QEMUFile *mig_read)
> +{
> +int fds[2];
> +RAMBlock *block;
> +
> +DPRINTF("fork\n");
> +
> +/* socketpair(AF_UNIX)? */
> +
> +if (qemu_pipe(fds) == -1) {
> +perror("qemu_pipe");
> +abort();
> +}
> +state.from_umemd_fd = fds[0];
> +umemd.to_qemu_fd = fds[1];
> +
> +if (qemu_pipe(fds) == -1) {
> +perror("qemu_pipe");
> +abort();
> +}
> +umemd.from_qemu_fd = fds[0];
> +state.to_umemd_fd = fds[1];
> +
> +pid_t child = fork();
> +if (child < 0) {
> +perror("fork");
> +abort();
> +}
> +
> +if (child == 0) {
> +int mig_write_fd;
> +
> +fd_close(&state.to_umemd_fd);
> +fd_close(&state.from_umemd_fd);
> +umemd.host_page_size = state.host_page_size;
> +umemd.host_page_shift = state.host_page_shift;
> +
> +umemd.nr_host_pages_per_target_page =
> +TARGET_PAGE_SIZE / umemd.host_page_size;
> +umemd.nr_target_pages_per_host_page =
> +umemd.host_page_size / TARGET_PAGE_SIZE;
> +
> +umemd.target_to_host_page_shift =
> +ffs(umemd.nr_host_pages_per_target_page) - 1;
> +umemd.host_to_target_page_shift =
> +ffs(umemd.nr_target_pages_per_host_page) - 1;
> +
> +umemd.state = 0;
> +umemd.version_id = state.version_id;
> +umemd.mig_read_fd = mig_read_fd;
> +umemd.mig_read = mig_read;
> +
> +mig_write_fd = dup(mig_read_fd);
> +if (mig_write_fd < 0) {
> +perror("could not dup for writable socket \n");
> +abort();
> +}
> +umemd.mig_write_fd = mig_write_fd;
> +umemd.mig_write = qemu_fopen_nonblock(mig_write_fd);
> +
> +postcopy_incoming_umemd(); /* noreturn */
> +}
> +
> +DPRINTF("qemu pid: %d daemon pid: %d\n", getpid(), child);
> +fd_close(&umemd.to_qemu_fd);
> +fd_close(&umemd.from_qemu_fd);
> +state.faulted_pages = g_malloc(umem_pages_size(MAX_FAULTED_PAG

[PATCH v2 36/41] postcopy: implement incoming part of postcopy live migration

[Isaku Yamahata]

This patch implements postcopy live migration for incoming part

Signed-off-by: Isaku Yamahata 
---
Changes v3 -> v4:
- fork umemd early to address qemu devices touching guest ram via
  post/pre_load
- code clean up on initialization
- Makefile.target
  migration-postcopy.c is target dependent due to TARGET_PAGE_xxx
  So it can't be shared between target architecture.
- use qemu_fopen_fd
- introduce incoming_flags_use_umem_make_present flag
- use MADV_DONTNEED

Changes v2 -> v3:
- make incoming socket nonblocking
- several clean ups
- Dropped QEMUFilePipe
- Moved QEMUFileNonblock to buffered_file
- Split out into umem/incoming/outgoing

Changes v1 -> v2:
- make mig_read nonblocking when socket
- updates for umem device changes
---
 Makefile.target|5 +
 cpu-all.h  |7 +
 exec.c |   20 +-
 migration-exec.c   |4 +
 migration-fd.c |6 +
 .../linux/umem.h => migration-postcopy-stub.c  |   47 +-
 migration-postcopy.c   | 1267 
 migration.c|4 +
 migration.h|   13 +
 qemu-common.h  |1 +
 qemu-options.hx|5 +-
 savevm.c   |   43 +
 vl.c   |8 +-
 13 files changed, 1409 insertions(+), 21 deletions(-)
 copy linux-headers/linux/umem.h => migration-postcopy-stub.c (55%)
 create mode 100644 migration-postcopy.c

diff --git a/Makefile.target b/Makefile.target
index 1582904..618bd3e 100644
--- a/Makefile.target
+++ b/Makefile.target
@@ -4,6 +4,7 @@ GENERATED_HEADERS = config-target.h
 CONFIG_NO_PCI = $(if $(subst n,,$(CONFIG_PCI)),n,y)
 CONFIG_NO_KVM = $(if $(subst n,,$(CONFIG_KVM)),n,y)
 CONFIG_NO_XEN = $(if $(subst n,,$(CONFIG_XEN)),n,y)
+CONFIG_NO_POSTCOPY = $(if $(subst n,,$(CONFIG_POSTCOPY)),n,y)
 
 include ../config-host.mak
 include config-devices.mak
@@ -196,6 +197,10 @@ LIBS+=-lz
 
 obj-i386-$(CONFIG_KVM) += hyperv.o
 
+obj-$(CONFIG_POSTCOPY) += migration-postcopy.o
+obj-$(CONFIG_NO_POSTCOPY) += migration-postcopy-stub.o
+common-obj-$(CONFIG_POSTCOPY) += umem.o
+
 QEMU_CFLAGS += $(VNC_TLS_CFLAGS)
 QEMU_CFLAGS += $(VNC_SASL_CFLAGS)
 QEMU_CFLAGS += $(VNC_JPEG_CFLAGS)
diff --git a/cpu-all.h b/cpu-all.h
index ff7f827..e0956bc 100644
--- a/cpu-all.h
+++ b/cpu-all.h
@@ -486,6 +486,9 @@ extern ram_addr_t ram_size;
 /* RAM is pre-allocated and passed into qemu_ram_alloc_from_ptr */
 #define RAM_PREALLOC_MASK   (1 << 0)
 
+/* RAM is allocated via umem for postcopy incoming mode */
+#define RAM_POSTCOPY_UMEM_MASK  (1 << 1)
+
 typedef struct RAMBlock {
 struct MemoryRegion *mr;
 uint8_t *host;
@@ -497,6 +500,10 @@ typedef struct RAMBlock {
 #if defined(__linux__) && !defined(TARGET_S390X)
 int fd;
 #endif
+
+#ifdef CONFIG_POSTCOPY
+UMem *umem;/* for incoming postcopy mode */
+#endif
 } RAMBlock;
 
 typedef struct RAMList {
diff --git a/exec.c b/exec.c
index 785..e5ff2ed 100644
--- a/exec.c
+++ b/exec.c
@@ -36,6 +36,7 @@
 #include "arch_init.h"
 #include "memory.h"
 #include "exec-memory.h"
+#include "migration.h"
 #if defined(CONFIG_USER_ONLY)
 #include 
 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
@@ -2632,6 +2633,13 @@ ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void 
*host,
 new_block->host = host;
 new_block->flags |= RAM_PREALLOC_MASK;
 } else {
+#ifdef CONFIG_POSTCOPY
+if (incoming_postcopy) {
+ram_addr_t page_size = getpagesize();
+size = (size + page_size - 1) & ~(page_size - 1);
+mem_path = NULL;
+}
+#endif
 if (mem_path) {
 #if defined (__linux__) && !defined(TARGET_S390X)
 new_block->host = file_ram_alloc(new_block, size, mem_path);
@@ -2709,7 +2717,13 @@ void qemu_ram_free(ram_addr_t addr)
 QLIST_REMOVE(block, next);
 if (block->flags & RAM_PREALLOC_MASK) {
 ;
-} else if (mem_path) {
+}
+#ifdef CONFIG_POSTCOPY
+else if (block->flags & RAM_POSTCOPY_UMEM_MASK) {
+postcopy_incoming_ram_free(block->umem);
+}
+#endif
+else if (mem_path) {
 #if defined (__linux__) && !defined(TARGET_S390X)
 if (block->fd) {
 munmap(block->host, block->length);
@@ -2755,6 +2769,10 @@ void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
 } else {
 flags = MAP_FIXED;
 munmap(vaddr, length);
+if (block->flags & RAM_POSTCOPY_UMEM_MASK) {
+postcopy_incoming_qemu_pages_unmapped(addr, length);
+block->flags &= ~RAM_POSTCOPY_UMEM_MASK;
+}