Re: [PATCH v2 35/41] postcopy: introduce helper functions for postcopy

[Juan Quintela]

Isaku Yamahata  wrote:
> On Thu, Jun 14, 2012 at 11:34:09PM +0200, Juan Quintela wrote:

>> > +size_t umem_pages_size(uint64_t nr)
>> > +{
>> > +return sizeof(struct umem_pages) + nr * sizeof(uint64_t);
>> 
>> Can we make sure that the pgoffs field is aligned?  I know that as it is
>> now it is aligned, but better to be sure?
>
> It is already done by gcc extension, zero length array.

Ah, I didn't knew that propierty of the zero arrays extension.  thanks.

>> 
>> Grr, we don't have a function that writes does a "safe_write".  The most
>> similar thing in qemu looks to be send_all().
>
> So we should introduce something like qemu_safe_write/read?

I guess so.  If you look around, you will see that we have a lot of
cases where we have this pattern.  But that is not a problem ofthis
patch, was already there.

>
>> Talking about looking, what protects that no other thread enters this
>> function before this one calls madvise?   Or I am losing something obvious?
>
> It is assumed that only main thread calls this function via iohandler.

Ok.  Can we add a comment then?

Later, Juan.
--
To unsubscribe from this list: send the line "unsubscribe kvm" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v2 35/41] postcopy: introduce helper functions for postcopy

[Isaku Yamahata]

On Thu, Jun 14, 2012 at 11:34:09PM +0200, Juan Quintela wrote:
> Isaku Yamahata  wrote:
> > +//#define DEBUG_UMEM
> > +#ifdef DEBUG_UMEM
> > +#include 
> > +#define DPRINTF(format, ...)\
> > +do {\
> > +printf("%d:%ld %s:%d "format, getpid(), syscall(SYS_gettid),\
> > +   __func__, __LINE__, ## __VA_ARGS__); \
> > +} while (0)
> 
> This should be in a header file that is linux specific?  And (at least
> on my systems) gettid is already defined on glibc.

I'll remove getpid/gettid. It was just for debugging in early phase.
They are not necessary any more.


> > +#else
> > +#define DPRINTF(format, ...)do { } while (0)
> > +#endif
> 
> 
> > +
> > +#define DEV_UMEM"/dev/umem"
> > +
> > +UMem *umem_new(void *hostp, size_t size)
> > +{
> > +struct umem_init uinit = {
> > +.size = size,
> > +};
> > +UMem *umem;
> > +
> > +assert((size % getpagesize()) == 0);
> > +umem = g_new(UMem, 1);
> > +umem->fd = open(DEV_UMEM, O_RDWR);
> > +if (umem->fd < 0) {
> > +perror("can't open "DEV_UMEM);
> > +abort();
> 
> Can we return one error insntead of abort?  the same for the rest of the
> file aborts.

Ok.


> > +size_t umem_pages_size(uint64_t nr)
> > +{
> > +return sizeof(struct umem_pages) + nr * sizeof(uint64_t);
> 
> Can we make sure that the pgoffs field is aligned?  I know that as it is
> now it is aligned, but better to be sure?

It is already done by gcc extension, zero length array.


> > +}
> > +
> > +static void umem_write_cmd(int fd, uint8_t cmd)
> > +{
> > +DPRINTF("write cmd %c\n", cmd);
> > +
> > +for (;;) {
> > +ssize_t ret = write(fd, &cmd, 1);
> > +if (ret == -1) {
> > +if (errno == EINTR) {
> > +continue;
> > +} else if (errno == EPIPE) {
> > +perror("pipe");
> > +DPRINTF("write cmd %c %zd %d: pipe is closed\n",
> > +cmd, ret, errno);
> > +break;
> > +}
> 
> 
> Grr, we don't have a function that writes does a "safe_write".  The most
> similar thing in qemu looks to be send_all().

So we should introduce something like qemu_safe_write/read?


> > +
> > +perror("pipe");
> 
> Can we make a different perror() message than previous error?
> 
> > +DPRINTF("write cmd %c %zd %d\n", cmd, ret, errno);
> > +abort();
> > +}
> > +
> > +break;
> > +}
> > +}
> > +
> > +static void umem_read_cmd(int fd, uint8_t expect)
> > +{
> > +uint8_t cmd;
> > +for (;;) {
> > +ssize_t ret = read(fd, &cmd, 1);
> > +if (ret == -1) {
> > +if (errno == EINTR) {
> > +continue;
> > +}
> > +perror("pipe");
> > +DPRINTF("read error cmd %c %zd %d\n", cmd, ret, errno);
> > +abort();
> > +}
> > +
> > +if (ret == 0) {
> > +DPRINTF("read cmd %c %zd: pipe is closed\n", cmd, ret);
> > +abort();
> > +}
> > +
> > +break;
> > +}
> > +
> > +DPRINTF("read cmd %c\n", cmd);
> > +if (cmd != expect) {
> > +DPRINTF("cmd %c expect %d\n", cmd, expect);
> > +abort();
> 
> Ouch.  If we receive garbage, we just exit?
> 
> I really think that we should implement error handling.
> 
> > +}
> > +}
> > +
> > +struct umem_pages *umem_recv_pages(QEMUFile *f, int *offset)
> > +{
> > +int ret;
> > +uint64_t nr;
> > +size_t size;
> > +struct umem_pages *pages;
> > +
> > +ret = qemu_peek_buffer(f, (uint8_t*)&nr, sizeof(nr), *offset);
> > +*offset += sizeof(nr);
> > +DPRINTF("ret %d nr %ld\n", ret, nr);
> > +if (ret != sizeof(nr) || nr == 0) {
> > +return NULL;
> > +}
> > +
> > +size = umem_pages_size(nr);
> > +pages = g_malloc(size);
> 
> Just thinking about this.  Couldn't we just decide on a "big enough"
> buffer, and never send anything bigger than that?  That would remove the
> need to have to malloc()/free() a buffer for each reception?

Will try to address it.


> > +/* qemu side handler */
> > +struct umem_pages *umem_qemu_trigger_page_fault(QEMUFile *from_umemd,
> > +int *offset)
> > +{
> > +uint64_t i;
> > +int page_shift = ffs(getpagesize()) - 1;
> > +struct umem_pages *pages = umem_recv_pages(from_umemd, offset);
> > +if (pages == NULL) {
> > +return NULL;
> > +}
> > +
> > +for (i = 0; i < pages->nr; i++) {
> > +ram_addr_t addr = pages->pgoffs[i] << page_shift;
> > +
> > +/* make pages present by forcibly triggering page fault. */
> > +volatile uint8_t *ram = qemu_get_ram_ptr(addr);
> > +uint8_t dummy_read = ram[0];
> > +(void)dummy_read;   /* suppress unused variable warning */
> > +  

Re: [PATCH v2 35/41] postcopy: introduce helper functions for postcopy

[Juan Quintela]

Isaku Yamahata  wrote:
> +//#define DEBUG_UMEM
> +#ifdef DEBUG_UMEM
> +#include 
> +#define DPRINTF(format, ...)\
> +do {\
> +printf("%d:%ld %s:%d "format, getpid(), syscall(SYS_gettid),\
> +   __func__, __LINE__, ## __VA_ARGS__); \
> +} while (0)

This should be in a header file that is linux specific?  And (at least
on my systems) gettid is already defined on glibc.


> +#else
> +#define DPRINTF(format, ...)do { } while (0)
> +#endif


> +
> +#define DEV_UMEM"/dev/umem"
> +
> +UMem *umem_new(void *hostp, size_t size)
> +{
> +struct umem_init uinit = {
> +.size = size,
> +};
> +UMem *umem;
> +
> +assert((size % getpagesize()) == 0);
> +umem = g_new(UMem, 1);
> +umem->fd = open(DEV_UMEM, O_RDWR);
> +if (umem->fd < 0) {
> +perror("can't open "DEV_UMEM);
> +abort();

Can we return one error insntead of abort?  the same for the rest of the
file aborts.


> +size_t umem_pages_size(uint64_t nr)
> +{
> +return sizeof(struct umem_pages) + nr * sizeof(uint64_t);

Can we make sure that the pgoffs field is aligned?  I know that as it is
now it is aligned, but better to be sure?

> +}
> +
> +static void umem_write_cmd(int fd, uint8_t cmd)
> +{
> +DPRINTF("write cmd %c\n", cmd);
> +
> +for (;;) {
> +ssize_t ret = write(fd, &cmd, 1);
> +if (ret == -1) {
> +if (errno == EINTR) {
> +continue;
> +} else if (errno == EPIPE) {
> +perror("pipe");
> +DPRINTF("write cmd %c %zd %d: pipe is closed\n",
> +cmd, ret, errno);
> +break;
> +}


Grr, we don't have a function that writes does a "safe_write".  The most
similar thing in qemu looks to be send_all().

> +
> +perror("pipe");

Can we make a different perror() message than previous error?

> +DPRINTF("write cmd %c %zd %d\n", cmd, ret, errno);
> +abort();
> +}
> +
> +break;
> +}
> +}
> +
> +static void umem_read_cmd(int fd, uint8_t expect)
> +{
> +uint8_t cmd;
> +for (;;) {
> +ssize_t ret = read(fd, &cmd, 1);
> +if (ret == -1) {
> +if (errno == EINTR) {
> +continue;
> +}
> +perror("pipe");
> +DPRINTF("read error cmd %c %zd %d\n", cmd, ret, errno);
> +abort();
> +}
> +
> +if (ret == 0) {
> +DPRINTF("read cmd %c %zd: pipe is closed\n", cmd, ret);
> +abort();
> +}
> +
> +break;
> +}
> +
> +DPRINTF("read cmd %c\n", cmd);
> +if (cmd != expect) {
> +DPRINTF("cmd %c expect %d\n", cmd, expect);
> +abort();

Ouch.  If we receive garbage, we just exit?

I really think that we should implement error handling.

> +}
> +}
> +
> +struct umem_pages *umem_recv_pages(QEMUFile *f, int *offset)
> +{
> +int ret;
> +uint64_t nr;
> +size_t size;
> +struct umem_pages *pages;
> +
> +ret = qemu_peek_buffer(f, (uint8_t*)&nr, sizeof(nr), *offset);
> +*offset += sizeof(nr);
> +DPRINTF("ret %d nr %ld\n", ret, nr);
> +if (ret != sizeof(nr) || nr == 0) {
> +return NULL;
> +}
> +
> +size = umem_pages_size(nr);
> +pages = g_malloc(size);

Just thinking about this.  Couldn't we just decide on a "big enough"
buffer, and never send anything bigger than that?  That would remove the
need to have to malloc()/free() a buffer for each reception?



> +/* qemu side handler */
> +struct umem_pages *umem_qemu_trigger_page_fault(QEMUFile *from_umemd,
> +int *offset)
> +{
> +uint64_t i;
> +int page_shift = ffs(getpagesize()) - 1;
> +struct umem_pages *pages = umem_recv_pages(from_umemd, offset);
> +if (pages == NULL) {
> +return NULL;
> +}
> +
> +for (i = 0; i < pages->nr; i++) {
> +ram_addr_t addr = pages->pgoffs[i] << page_shift;
> +
> +/* make pages present by forcibly triggering page fault. */
> +volatile uint8_t *ram = qemu_get_ram_ptr(addr);
> +uint8_t dummy_read = ram[0];
> +(void)dummy_read;   /* suppress unused variable warning */
> +}
> +
> +/*
> + * Very Linux implementation specific.
> + * Make it sure that other thread doesn't fault on the above virtual
> + * address. (More exactly other thread doesn't call fault handler with
> + * the offset.)
> + * the fault handler is called with mmap_sem read locked.
> + * madvise() does down/up_write(mmap_sem)
> + */
> +qemu_madvise(NULL, 0, MADV_NORMAL);

If it is linux specific, should be inside CONFIG_LINUX ifdef, or a
function hided on some header.

Talking about looking, what protects that no other thread enters this
function before this one calls mad

[PATCH v2 35/41] postcopy: introduce helper functions for postcopy

[Isaku Yamahata]

This patch introduces helper function for postcopy to access
umem char device and to communicate between incoming-qemu and umemd.

Signed-off-by: Isaku Yamahata 
---
changes v1 -> v2:
- code simplification
- make fault trigger more robust
- introduce struct umem_pages
---
 umem.c |  364 
 umem.h |  101 ++
 2 files changed, 465 insertions(+), 0 deletions(-)
 create mode 100644 umem.c
 create mode 100644 umem.h

diff --git a/umem.c b/umem.c
new file mode 100644
index 000..64eaab5
--- /dev/null
+++ b/umem.c
@@ -0,0 +1,364 @@
+/*
+ * umem.c: user process backed memory module for postcopy livemigration
+ *
+ * Copyright (c) 2011
+ * National Institute of Advanced Industrial Science and Technology
+ *
+ * https://sites.google.com/site/grivonhome/quick-kvm-migration
+ * Author: Isaku Yamahata 
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see .
+ */
+
+#include 
+#include 
+
+#include 
+
+#include "bitops.h"
+#include "sysemu.h"
+#include "hw/hw.h"
+#include "umem.h"
+
+//#define DEBUG_UMEM
+#ifdef DEBUG_UMEM
+#include 
+#define DPRINTF(format, ...)\
+do {\
+printf("%d:%ld %s:%d "format, getpid(), syscall(SYS_gettid),\
+   __func__, __LINE__, ## __VA_ARGS__); \
+} while (0)
+#else
+#define DPRINTF(format, ...)do { } while (0)
+#endif
+
+#define DEV_UMEM"/dev/umem"
+
+UMem *umem_new(void *hostp, size_t size)
+{
+struct umem_init uinit = {
+.size = size,
+};
+UMem *umem;
+
+assert((size % getpagesize()) == 0);
+umem = g_new(UMem, 1);
+umem->fd = open(DEV_UMEM, O_RDWR);
+if (umem->fd < 0) {
+perror("can't open "DEV_UMEM);
+abort();
+}
+
+if (ioctl(umem->fd, UMEM_INIT, &uinit) < 0) {
+perror("UMEM_INIT");
+abort();
+}
+if (ftruncate(uinit.shmem_fd, uinit.size) < 0) {
+perror("truncate(\"shmem_fd\")");
+abort();
+}
+
+umem->nbits = 0;
+umem->nsets = 0;
+umem->faulted = NULL;
+umem->page_shift = ffs(getpagesize()) - 1;
+umem->shmem_fd = uinit.shmem_fd;
+umem->size = uinit.size;
+umem->umem = mmap(hostp, size, PROT_EXEC | PROT_READ | PROT_WRITE,
+  MAP_PRIVATE | MAP_FIXED, umem->fd, 0);
+if (umem->umem == MAP_FAILED) {
+perror("mmap(UMem) failed");
+abort();
+}
+return umem;
+}
+
+void umem_destroy(UMem *umem)
+{
+if (umem->fd != -1) {
+close(umem->fd);
+}
+if (umem->shmem_fd != -1) {
+close(umem->shmem_fd);
+}
+g_free(umem->faulted);
+g_free(umem);
+}
+
+void umem_get_page_request(UMem *umem, struct umem_pages *page_request)
+{
+ssize_t ret = read(umem->fd, page_request->pgoffs,
+   page_request->nr * sizeof(page_request->pgoffs[0]));
+if (ret < 0) {
+perror("daemon: umem read");
+abort();
+}
+page_request->nr = ret / sizeof(page_request->pgoffs[0]);
+}
+
+void umem_mark_page_cached(UMem *umem, struct umem_pages *page_cached)
+{
+const void *buf = page_cached->pgoffs;
+ssize_t left = page_cached->nr * sizeof(page_cached->pgoffs[0]);
+
+while (left > 0) {
+ssize_t ret = write(umem->fd, buf, left);
+if (ret == -1) {
+if (errno == EINTR)
+continue;
+
+perror("daemon: umem write");
+abort();
+}
+
+left -= ret;
+buf += ret;
+}
+}
+
+void umem_unmap(UMem *umem)
+{
+munmap(umem->umem, umem->size);
+umem->umem = NULL;
+}
+
+void umem_close(UMem *umem)
+{
+close(umem->fd);
+umem->fd = -1;
+}
+
+void *umem_map_shmem(UMem *umem)
+{
+umem->nbits = umem->size >> umem->page_shift;
+umem->nsets = 0;
+umem->faulted = g_new0(unsigned long, BITS_TO_LONGS(umem->nbits));
+
+umem->shmem = mmap(NULL, umem->size, PROT_READ | PROT_WRITE, MAP_SHARED,
+   umem->shmem_fd, 0);
+if (umem->shmem == MAP_FAILED) {
+perror("daemon: mmap(\"shmem\")");
+abort();
+}
+return umem->shmem;
+}
+
+void umem_unmap_shmem(UMem *umem)
+{
+munmap(umem->shmem, umem->size);
+umem->shmem = NULL;
+}
+
+void umem_remove_shmem(UMem *umem, size_t offset, size_t size)
+{
+int s = offset >> umem->page_shift;
+i