Ping? P.S. Please note a typo in the cover letter: when I referred to KSM, I meant "kernel samepage merging", not "kernel shared memory" :-)
On Tue, Nov 20, 2012 at 7:10 AM, Peter Feiner <pe...@gridcentric.ca> wrote: > This patch makes the -mem-path filenames deterministic and allows some > control > over how QEMU mmaps the files. Given this control, QEMU can be used to > implement > exogenous memory management techniques quite simply. Two examples follow: > > 1. Post-copy migration (mmap=shared for origin, save device state, > path in > NFS available on both hosts, mmap=shared for destination). > 2. memory sharing via VM cloning (mmap=shared for master, save device > state, > then mmap=private for clones). > > These new features are exposed via arguments to -mem-path: > > -mem-path \ > path[,mode=temp|open|create][,mmap=private|shared][,nofallback][,anyfs] > > The backing file names are made deterministic by including their RAMBlocks' > offsets, which are unique given a qemu command line. Note that Xen's live > migration relies on RAMBlocks having the same offsets between QEMU > processes > (see xen_read_physmap). The new file name format is > > qemu_back_mem.OFFSET+SIZE.NAME[.RANDOM] > > where SIZE and NAME are added for extra sanity checking when mode="open". > > By default, the old -mem-path behavior is preserved. I.e., mode="temp" is > used, > which adds a random suffix to the deterministic name and unlinks the files. > > Signed-off-by: Peter Feiner <pe...@gridcentric.ca> > --- > cpu-all.h | 5 ++++ > exec.c | 60 > ++++++++++++++++++++++++++++++++++-------------------- > qemu-config.c | 26 +++++++++++++++++++++++ > qemu-options.hx | 24 +++++++++++++++++++-- > vl.c | 43 +++++++++++++++++++++++++++++++++++++- > 5 files changed, 131 insertions(+), 27 deletions(-) > > diff --git a/cpu-all.h b/cpu-all.h > index c9c51b8..a83f71e 100644 > --- a/cpu-all.h > +++ b/cpu-all.h > @@ -500,6 +500,11 @@ typedef struct RAMList { > extern RAMList ram_list; > > extern const char *mem_path; > +extern int mem_temp; > +extern int mem_create; > +extern int mem_fallback; > +extern int mem_shared; > +extern int mem_hugetlbfs; > extern int mem_prealloc; > > /* Flags stored in the low bits of the TLB virtual address. These are > diff --git a/exec.c b/exec.c > index 8435de0..75b146c 100644 > --- a/exec.c > +++ b/exec.c > @@ -2346,7 +2346,7 @@ void qemu_flush_coalesced_mmio_buffer(void) > > #define HUGETLBFS_MAGIC 0x958458f6 > > -static long gethugepagesize(const char *path) > +static long getfspagesize(const char *path, int want_hugetlbfs) > { > struct statfs fs; > int ret; > @@ -2360,7 +2360,7 @@ static long gethugepagesize(const char *path) > return 0; > } > > - if (fs.f_type != HUGETLBFS_MAGIC) > + if (want_hugetlbfs && fs.f_type != HUGETLBFS_MAGIC) > fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path); > > return fs.f_bsize; > @@ -2373,17 +2373,15 @@ static void *file_ram_alloc(RAMBlock *block, > char *filename; > void *area; > int fd; > -#ifdef MAP_POPULATE > int flags; > -#endif > - unsigned long hpagesize; > + unsigned long pagesize; > > - hpagesize = gethugepagesize(path); > - if (!hpagesize) { > + pagesize = getfspagesize(path, mem_hugetlbfs); > + if (!pagesize) { > return NULL; > } > > - if (memory < hpagesize) { > + if (memory < pagesize) { > return NULL; > } > > @@ -2392,20 +2390,35 @@ static void *file_ram_alloc(RAMBlock *block, > return NULL; > } > > - if (asprintf(&filename, "%s/qemu_back_mem.XXXXXX", path) == -1) { > + if (asprintf(&filename, "%s/qemu_back_mem.%"PRIx64"+%"PRIx64".%s", > + path, block->offset, memory, block->mr->name) == -1) { > return NULL; > } > > - fd = mkstemp(filename); > + if (mem_temp) { > + if (asprintf(&filename, "%s.XXXXXX", filename) == -1) { > + return NULL; > + } > + fd = mkstemp(filename); > + } else { > + flags = O_RDWR | O_CLOEXEC | (mem_create ? O_CREAT : 0); > + fd = open(filename, flags, 0600); > + } > + > if (fd < 0) { > - perror("unable to create backing store for hugepages"); > + fprintf(stderr, "unable to create backing store %s " > + "for guest memory: %s\n", filename, strerror(errno)); > free(filename); > return NULL; > } > - unlink(filename); > + > + if (mem_temp) { > + unlink(filename); > + } > + > free(filename); > > - memory = (memory+hpagesize-1) & ~(hpagesize-1); > + memory = (memory+pagesize-1) & ~(pagesize-1); > > /* > * ftruncate is not supported by hugetlbfs in older > @@ -2416,16 +2429,16 @@ static void *file_ram_alloc(RAMBlock *block, > if (ftruncate(fd, memory)) > perror("ftruncate"); > > + flags = mem_shared ? MAP_SHARED : MAP_PRIVATE; > #ifdef MAP_POPULATE > /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the > case > - * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED > + * MAP_PRIVATE is requested. For mem_prealloc we always mmap as > MAP_SHARED > * to sidestep this quirk. > */ > - flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE; > - area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0); > -#else > - area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); > + if (mem_prealloc) > + flags = MAP_POPULATE | MAP_SHARED; > #endif > + area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0); > if (area == MAP_FAILED) { > perror("file_ram_alloc: can't mmap RAM pages"); > close(fd); > @@ -2561,6 +2574,10 @@ ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, > void *host, > #if defined (__linux__) && !defined(TARGET_S390X) > new_block->host = file_ram_alloc(new_block, size, mem_path); > if (!new_block->host) { > + if (!mem_fallback) { > + fprintf(stderr, "failed to allocate ram file for > %s\n", mr->name); > + exit(1); > + } > new_block->host = qemu_vmalloc(size); > memory_try_enable_merging(new_block->host, size); > } > @@ -2675,11 +2692,10 @@ void qemu_ram_remap(ram_addr_t addr, ram_addr_t > length) > if (mem_path) { > #if defined(__linux__) && !defined(TARGET_S390X) > if (block->fd) { > + flags = mem_shared ? MAP_SHARED : MAP_PRIVATE; > #ifdef MAP_POPULATE > - flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED > : > - MAP_PRIVATE; > -#else > - flags |= MAP_PRIVATE; > + if (mem_prealloc) > + flags = MAP_POPULATE | MAP_SHARED; > #endif > area = mmap(vaddr, length, PROT_READ | PROT_WRITE, > flags, block->fd, offset); > diff --git a/qemu-config.c b/qemu-config.c > index 10d1ba4..45b24d2 100644 > --- a/qemu-config.c > +++ b/qemu-config.c > @@ -691,6 +691,31 @@ static QemuOptsList qemu_object_opts = { > }, > }; > > +static QemuOptsList qemu_mempath_opts = { > + .name = "mempath", > + .implied_opt_name = "path", > + .head = QTAILQ_HEAD_INITIALIZER(qemu_mempath_opts.head), > + .desc = { > + { > + .name = "path", > + .type = QEMU_OPT_STRING, > + },{ > + .name = "mmap", > + .type = QEMU_OPT_STRING, > + },{ > + .name = "mode", > + .type = QEMU_OPT_STRING, > + },{ > + .name = "anyfs", > + .type = QEMU_OPT_BOOL, > + },{ > + .name = "fallback", > + .type = QEMU_OPT_BOOL, > + }, > + { /* end of list */ } > + }, > +}; > + > static QemuOptsList *vm_config_groups[32] = { > &qemu_drive_opts, > &qemu_chardev_opts, > @@ -709,6 +734,7 @@ static QemuOptsList *vm_config_groups[32] = { > &qemu_sandbox_opts, > &qemu_add_fd_opts, > &qemu_object_opts, > + &qemu_mempath_opts, > NULL, > }; > > diff --git a/qemu-options.hx b/qemu-options.hx > index 9bb29d3..8972397 100644 > --- a/qemu-options.hx > +++ b/qemu-options.hx > @@ -443,10 +443,28 @@ gigabytes respectively. > ETEXI > > DEF("mem-path", HAS_ARG, QEMU_OPTION_mempath, > - "-mem-path FILE provide backing storage for guest RAM\n", > QEMU_ARCH_ALL) > + "-mem-path > path[,mmap=private|shared][,mode=temp|open|create][,nofallback][,anyfs]\n" > + " provide backing storage for guest RAM\n", > + QEMU_ARCH_ALL) > STEXI > -@item -mem-path @var{path} > -Allocate guest RAM from a temporarily created file in @var{path}. > +@item -mem-path > @var{path}[,mmap=private|shared][,mode=temp|open|create][,nofallback][,anyfs] > +Allocate guest RAM from files in @var{path}. > +@item mmap=private|shared > +Specifies how backing files are mmap'd. "private" indicates a COW > mapping, thus > +leaving the underlying file unchanged. "shared" indicates a write-through > +mapping, thus reflecting the guest's memory in the underlying file. > Default is > +private. > +@item mode=temp|open|create > +Determines how backing files are created and opened. "temp" indicates that > +unique temporary files are created in @var{path} and unlinked after > opening. > +Both "create" and "open" indicate that deterministic names are used and > the > +files aren't unlinked. The The difference between "create" and "open" is > that > +"open" expects the files to already exist. Default is temp. > +@item nofallback > +Fail if backing files cannot be used for guest RAM (e.g., permission > error, bad > +path). By default, RAM allocation falls back to normal method. > +@item anyfs > +Suppressing warnings if @var{path} isn't on a hugetlbfs filesystem. > ETEXI > > #ifdef MAP_POPULATE > diff --git a/vl.c b/vl.c > index c8e9c78..3b7cea7 100644 > --- a/vl.c > +++ b/vl.c > @@ -185,6 +185,11 @@ static int display_remote; > const char* keyboard_layout = NULL; > ram_addr_t ram_size; > const char *mem_path = NULL; > +int mem_shared = 0; > +int mem_hugetlbfs = 1; > +int mem_temp = 1; > +int mem_create = 1; > +int mem_fallback = 1; > #ifdef MAP_POPULATE > int mem_prealloc = 0; /* force preallocation of physical target memory */ > #endif > @@ -2938,9 +2943,43 @@ int main(int argc, char **argv, char **envp) > } > break; > } > - case QEMU_OPTION_mempath: > - mem_path = optarg; > + case QEMU_OPTION_mempath: { > + const char *value; > + opts = qemu_opts_parse(qemu_find_opts("mempath"), optarg, > 1); > + if (!opts) { > + exit(1); > + } > + > + mem_path = qemu_opt_get(opts, "path"); > + mem_hugetlbfs = !qemu_opt_get_bool(opts, "anyfs", 0); > + mem_fallback = qemu_opt_get_bool(opts, "fallback", 1); > + > + value = qemu_opt_get(opts, "mode"); > + if (value == NULL || !strcmp(value, "temp")) { > + mem_temp = 1; > + } else { > + mem_temp = 0; > + if (!strcmp(value, "create")) { > + mem_create = 1; > + } else if (!strcmp(value, "open")) { > + mem_create = 0; > + } else { > + fprintf(stderr, "Invalid -mem-path mode > value.\n"); > + exit(1); > + } > + } > + > + value = qemu_opt_get(opts, "mmap"); > + if (value == NULL || !strcmp(value, "private")) { > + mem_shared = 0; > + } else if (!strcmp(value, "shared")) { > + mem_shared = 1; > + } else { > + fprintf(stderr, "Invalid -mem-path mmap value.\n"); > + exit(1); > + } > break; > + } > #ifdef MAP_POPULATE > case QEMU_OPTION_mem_prealloc: > mem_prealloc = 1; > -- > 1.7.5.4 > >