* Sanidhya Kashyap (sanidhya.ii...@gmail.com) wrote: > In this patch, I have incorporated an enum named QemuProcess > which defines what kind of process is being executed i.e. > none --> no other process except the VM execution > migration --> migration is being executed > bitmap-dump --> bitmap dump process is undergoing
OK, so that's the right idea; I suspect 'process' is probably the wrong word - and we use it to mean too many things already. Why don't you stick with the 'dbu': instead of: static QemuProcess dbu = QEMU_PROCESS_NONE; I'd use: static QemuDirtyBitmapUser dbu = QEMU_DBU_NONE; doesn't look too bad to me, and is more explicit about what it's for. > Besides this, I have tried to incorporate the dynamic change of > the last_ram_offset, if it gets change. The downside is that I am > holding lock for a longer period of time and I don't know whether > that should be done or not. I am also doing some allocation when > locked. > I am not sure whether last_ram_offset gets changed when a device > is hot plugged or hot unplugged. > > I have modified the variables name as: > current-iteration: for the current iteration under process > iterations: total iterations that will be done which is constant > period: the delay in each iteration. > > Signed-off-by: Sanidhya Kashyap <sanidhya.ii...@gmail.com> > --- > hmp-commands.hx | 16 ++ > hmp.c | 18 +++ > hmp.h | 1 + > include/exec/cpu-all.h | 5 +- > include/sysemu/sysemu.h | 5 + > migration.c | 12 ++ > qapi-schema.json | 35 +++++ > qmp-commands.hx | 34 +++++ > savevm.c | 378 > ++++++++++++++++++++++++++++++++++++++++++++++++ > vl.c | 24 +++ > 10 files changed, 527 insertions(+), 1 deletion(-) > > diff --git a/hmp-commands.hx b/hmp-commands.hx > index d0943b1..30b553e 100644 > --- a/hmp-commands.hx > +++ b/hmp-commands.hx > @@ -1788,6 +1788,22 @@ STEXI > show available trace events and their state > ETEXI > > + { > + .name = "ldb|log_dirty_bitmap", > + .args_type = "filename:s,iterations:i?,period:i?", > + .params = "filename iterations period", > + .help = "dumps the memory's dirty bitmap to file\n\t\t\t" > + "filename: name of the file in which the bitmap will > be saved\n\t\t\t" > + "iterations: number of times, the memory will be > logged\n\t\t\t" > + "period: time difference in milliseconds between each > iteration", > + .mhandler.cmd = hmp_log_dirty_bitmap, > + }, > +STEXI > +@item ldb or log_dirty_bitmap @var{filename} > +@findex log_dirty_bitmap > +dumps the writable working set of a VM's memory to a file > +ETEXI > + > STEXI > @end table > ETEXI > diff --git a/hmp.c b/hmp.c > index 4d1838e..d067420 100644 > --- a/hmp.c > +++ b/hmp.c > @@ -1318,6 +1318,24 @@ void hmp_device_del(Monitor *mon, const QDict *qdict) > hmp_handle_error(mon, &err); > } > > +void hmp_log_dirty_bitmap(Monitor *mon, const QDict *qdict) > +{ > + const char *filename = qdict_get_str(qdict, "filename"); > + bool has_iterations = qdict_haskey(qdict, "iterations"); > + int64_t iterations = qdict_get_try_int(qdict, "iterations", 3); > + bool has_period = qdict_haskey(qdict, "period"); > + int64_t period = qdict_get_try_int(qdict, "period", 10); > + Error *err = NULL; > + > + qmp_log_dirty_bitmap(filename, has_iterations, iterations, > + has_period, period, &err); > + if (err) { > + monitor_printf(mon, "log_dirty_bitmap: %s\n", error_get_pretty(err)); > + error_free(err); > + return; > + } > +} > + > void hmp_dump_guest_memory(Monitor *mon, const QDict *qdict) > { > Error *err = NULL; > diff --git a/hmp.h b/hmp.h > index 4fd3c4a..0895182 100644 > --- a/hmp.h > +++ b/hmp.h > @@ -94,6 +94,7 @@ void hmp_cpu_add(Monitor *mon, const QDict *qdict); > void hmp_object_add(Monitor *mon, const QDict *qdict); > void hmp_object_del(Monitor *mon, const QDict *qdict); > void hmp_info_memdev(Monitor *mon, const QDict *qdict); > +void hmp_log_dirty_bitmap(Monitor *mon, const QDict *qdict); > void object_add_completion(ReadLineState *rs, int nb_args, const char *str); > void object_del_completion(ReadLineState *rs, int nb_args, const char *str); > void device_add_completion(ReadLineState *rs, int nb_args, const char *str); > diff --git a/include/exec/cpu-all.h b/include/exec/cpu-all.h > index f91581f..179fc5b 100644 > --- a/include/exec/cpu-all.h > +++ b/include/exec/cpu-all.h > @@ -297,13 +297,16 @@ CPUArchState *cpu_copy(CPUArchState *env); > > /* memory API */ > > +/* global name which is used with both migration and bitmap dump */ > +#define RAMBLOCK_NAME_LENGTH 256 > + > typedef struct RAMBlock { > struct MemoryRegion *mr; > uint8_t *host; > ram_addr_t offset; > ram_addr_t length; > uint32_t flags; > - char idstr[256]; > + char idstr[RAMBLOCK_NAME_LENGTH]; > /* Reads can take either the iothread or the ramlist lock. > * Writes must take both locks. > */ > diff --git a/include/sysemu/sysemu.h b/include/sysemu/sysemu.h > index d8539fd..304a3e1 100644 > --- a/include/sysemu/sysemu.h > +++ b/include/sysemu/sysemu.h > @@ -227,4 +227,9 @@ extern QemuOptsList qemu_net_opts; > extern QemuOptsList qemu_global_opts; > extern QemuOptsList qemu_mon_opts; > > +/* migration vs dirty bitmap process */ > +bool qemu_process_check(QemuProcess user); > +void qemu_process_set(QemuProcess user); > +const char *get_qemu_process_as_string(void); > + > #endif > diff --git a/migration.c b/migration.c > index 8d675b3..7b61b1e 100644 > --- a/migration.c > +++ b/migration.c > @@ -117,6 +117,7 @@ static void process_incoming_migration_co(void *opaque) > } else { > runstate_set(RUN_STATE_PAUSED); > } > + qemu_process_set(QEMU_PROCESS_NONE); > } > > void process_incoming_migration(QEMUFile *f) > @@ -317,6 +318,7 @@ static void migrate_fd_cleanup(void *opaque) > } > > notifier_list_notify(&migration_state_notifiers, s); > + qemu_process_set(QEMU_PROCESS_NONE); > } > > void migrate_fd_error(MigrationState *s) > @@ -326,6 +328,7 @@ void migrate_fd_error(MigrationState *s) > s->state = MIG_STATE_ERROR; > trace_migrate_set_state(MIG_STATE_ERROR); > notifier_list_notify(&migration_state_notifiers, s); > + qemu_process_set(QEMU_PROCESS_NONE); > } > > static void migrate_fd_cancel(MigrationState *s) > @@ -436,6 +439,15 @@ void qmp_migrate(const char *uri, bool has_blk, bool blk, > return; > } > > + if (!qemu_process_check(QEMU_PROCESS_NONE) && > + !qemu_process_check(QEMU_PROCESS_MIGRATION)) { > + error_setg(errp, "Migration not possible, since %s " > + "is in progress.\n", get_qemu_process_as_string()); Don't need that '\n' > + return; > + } > + > + qemu_process_set(QEMU_PROCESS_MIGRATION); > + > s = migrate_init(¶ms); > > if (strstart(uri, "tcp:", &p)) { > diff --git a/qapi-schema.json b/qapi-schema.json > index b11aad2..dced3c2 100644 > --- a/qapi-schema.json > +++ b/qapi-schema.json > @@ -3480,3 +3480,38 @@ > # Since: 2.1 > ## > { 'command': 'rtc-reset-reinjection' } > + > +## > +# QemuProcess > +# > +# @none: no other process is being executed besides a simple VM execution. > +# > +# @migration: migration process is going on. > +# > +# @bitmap-dump: bitmap dump process is being executed. > +# > +# Since 2.2 > +## > +{ 'enum': 'QemuProcess', > + 'data': [ 'none', 'migration', 'bitmap-dump' ] } > + > +## > +# @log-dirty-bitmap > +# > +# This command will dump the dirty bitmap to a file by logging the > +# memory for a specified number of times with a defined time difference > +# > +# @filename: name of the file in which the bitmap will be saved. > +# > +# @iterations: number of times the memory will be logged (optional). The > +# and max values are 3 and 100000 respectively. > +# > +# @period: time difference in milliseconds between each iteration (optional). > +# The min and max values are 10 and 100000 respectively. > +# > +# Since 2.2 > +## > +{ 'command' : 'log-dirty-bitmap', > + 'data' : { 'filename' : 'str', > + '*iterations' : 'int', > + '*period' : 'int' } } > diff --git a/qmp-commands.hx b/qmp-commands.hx > index 4be4765..2ead2ca 100644 > --- a/qmp-commands.hx > +++ b/qmp-commands.hx > @@ -3753,5 +3753,39 @@ Example: > > -> { "execute": "rtc-reset-reinjection" } > <- { "return": {} } > +EQMP > + > + { > + .name = "log-dirty-bitmap", > + .args_type = "filename:s,iterations:i?,period:i?", > + .mhandler.cmd_new = qmp_marshal_input_log_dirty_bitmap, > + }, > + > +SQMP > +log-dirty-bitmap > +---------------- > + > +start logging the memory of the VM for writable working set > + > +Arguments: > + > +- "filename": name of the file, in which the bitmap will be saved. > + > +- "iterations": number of times, the memory will be logged (optional). > + The min and max values are 3 and 100000 respectively. > + > +- "period": time difference in milliseconds between each iteration > (optional). > + The min and max values are 10 and 100000 respectively. > + > +Examples: > +-> { "execute": "log-dirty-bitmap", > + "arguments": { > + "filename": "/tmp/fileXXX", > + "iterations": 3, > + "period": 10 } } > + > +<- { "return": {} } > > +Note: The iterations, and period parameters are optional. iterations default > +value is 3 while that of period is 10. > EQMP > diff --git a/savevm.c b/savevm.c > index e19ae0a..125e5ed 100644 > --- a/savevm.c > +++ b/savevm.c > @@ -42,6 +42,9 @@ > #include "qemu/iov.h" > #include "block/snapshot.h" > #include "block/qapi.h" > +#include "exec/address-spaces.h" > +#include "exec/ram_addr.h" > +#include "qemu/bitmap.h" > > > #ifndef ETH_P_RARP > @@ -1137,6 +1140,381 @@ void do_savevm(Monitor *mon, const QDict *qdict) > } > } > > +/* > + * Adding the functionality of continuous logging of the > + * dirty bitmap which is almost similar to the migration > + * thread > + */ > + > +enum { > + LOG_BITMAP_STATE_ERROR = -1, > + LOG_BITMAP_STATE_NONE, > + LOG_BITMAP_STATE_ACTIVE, > + LOG_BITMAP_STATE_CANCELING, > + LOG_BITMAP_STATE_COMPLETED > +}; > + > +typedef struct BitmapLogState BitmapLogState; > +static int64_t MIN_ITERATION_VALUE = 3; > +static int64_t MIN_PERIOD_VALUE = 10; > +static int64_t MAX_ITERATION_VALUE = 100000; > +static int64_t MAX_PERIOD_VALUE = 100000; > + > +struct BitmapLogState { > + int state; > + int fd; > + int64_t current_period; > + int64_t current_iteration; > + int64_t iterations; > + unsigned long *log_bitmap_array; > + QemuThread thread; > +}; > + > +/* > + * helper functions > + */ > + > +static inline void log_bitmap_lock(void) > +{ > + qemu_mutex_lock_iothread(); > + qemu_mutex_lock_ramlist(); > +} > + > +static inline void log_bitmap_unlock(void) > +{ > + qemu_mutex_unlock_ramlist(); > + qemu_mutex_unlock_iothread(); > +} > + > +static inline void log_bitmap_set_dirty(ram_addr_t addr, > + unsigned long *log_bitmap_array) > +{ > + long nr = addr >> TARGET_PAGE_BITS; > + set_bit(nr, log_bitmap_array); > +} > + > +static bool log_bitmap_set_status(BitmapLogState *b, > + int old_state, > + int new_state) > +{ > + return atomic_cmpxchg(&b->state, old_state, new_state); > +} > + > +/* > + * inspired from migration mechanism > + */ > + > +static BitmapLogState *log_bitmap_get_current_state(void) > +{ > + static BitmapLogState current_bitmaplogstate = { > + .state = LOG_BITMAP_STATE_NONE, > + .log_bitmap_array = NULL, > + }; > + > + return ¤t_bitmaplogstate; > +} > + > +/* > + * syncing the log_bitmap with the ram_list dirty bitmap > + */ > + > +static void log_bitmap_dirty_bitmap_sync(unsigned long *log_bitmap_array) > +{ > + RAMBlock *block; > + uint64_t counter = 0; /* 0 means log bitmap */ > + address_space_sync_dirty_bitmap(&address_space_memory); > + QTAILQ_FOREACH(block, &ram_list.blocks, next) { > + qemu_bitmap_sync_range(block->mr->ram_addr, block->length, > + log_bitmap_array, &counter); > + } > +} > + > +static inline bool value_in_range(int64_t value, int64_t min_value, > + int64_t max_value, const char *str, > + Error **errp) > +{ > + if (value < min_value) { > + error_setg(errp, "%s's value must be greater than %ld", > + str, min_value); > + return false; > + } > + if (value > max_value) { > + error_setg(errp, "%s's value must be less than %ld", > + str, max_value); > + return false; > + } > + return true; > +} > + > +static inline void log_bitmap_close(BitmapLogState *b) > +{ > + log_bitmap_lock(); > + memory_global_dirty_log_stop(); > + log_bitmap_unlock(); > + > + g_free(b->log_bitmap_array); > + b->log_bitmap_array = NULL; > + qemu_close(b->fd); > + b->fd = -1; > +} > + > +static bool log_bitmap_ram_block_info_dump(int fd, int64_t ram_bitmap_pages, > + bool dump_blocks_info) > +{ > + int block_count = 0; > + int block_name_length; > + RAMBlock *block; > + int ret; > + > + if (qemu_write_full(fd, &ram_bitmap_pages, sizeof(int64_t)) < 0) { > + return true; > + } > + > + if (dump_blocks_info) { > + > + QTAILQ_FOREACH(block, &ram_list.blocks, next) { > + block_count++; > + } > + > + ret = qemu_write_full(fd, &block_count, sizeof(int)); > + if (ret < sizeof(int)) { > + return true; > + } > + > + QTAILQ_FOREACH(block, &ram_list.blocks, next) { > + block_name_length = strlen(block->idstr) + 1; > + ret = qemu_write_full(fd, &block_name_length, sizeof(int)); > + if (ret < sizeof(int)) { > + return true; > + } > + > + ret = qemu_write_full(fd, &(block->idstr), sizeof(char) * > + block_name_length); > + if (ret < sizeof(char) * block_name_length) { > + return true; > + } > + > + ret = qemu_write_full(fd, &(block->offset), sizeof(ram_addr_t)); > + if (ret < sizeof(ram_addr_t)) { > + return true; > + } > + > + ret = qemu_write_full(fd, &(block->length), sizeof(ram_addr_t)); > + if (ret < sizeof(ram_addr_t)) { > + return true; > + } > + } > + } > + return false; > +} > + > +static void log_bitmap_update_status(BitmapLogState *b) > +{ > + int s = b->state; > + switch (s) { > + case LOG_BITMAP_STATE_ACTIVE: > + case LOG_BITMAP_STATE_CANCELING: > + case LOG_BITMAP_STATE_ERROR: > + log_bitmap_set_status(b, s, LOG_BITMAP_STATE_COMPLETED); > + } > + return; > +} > + > +static void *bitmap_logging_thread(void *opaque) > +{ > + /* > + * setup basic structures > + */ > + > + BitmapLogState *b = opaque; > + int fd = b->fd; > + int64_t current_ram_bitmap_pages, prev_ram_bitmap_pages; > + size_t bitmap_size = 0; > + unsigned long *temp_log_bitmap_array = NULL; > + char marker = 'M'; > + int ret; > + > + b->current_iteration = 1; > + log_bitmap_set_status(b, LOG_BITMAP_STATE_NONE, > + LOG_BITMAP_STATE_ACTIVE); > + > + current_ram_bitmap_pages = 0; > + prev_ram_bitmap_pages = 1; > + > + /* > + * start the logging period > + */ > + > + /* > + * need lock for getting the information about the ram pages. > + * This does not change on acquiring the lock > + */ > + log_bitmap_lock(); > + current_ram_bitmap_pages = last_ram_offset() >> TARGET_PAGE_BITS; > + bitmap_size = BITS_TO_LONGS(current_ram_bitmap_pages) * > + sizeof(unsigned long); > + b->log_bitmap_array = bitmap_new(current_ram_bitmap_pages); > + if (b->log_bitmap_array == NULL) { > + b->state = LOG_BITMAP_STATE_ERROR; > + log_bitmap_unlock(); > + goto log_thread_end; > + } > + > + memory_global_dirty_log_start(); > + log_bitmap_dirty_bitmap_sync(b->log_bitmap_array); > + log_bitmap_unlock(); > + > + /* > + * sync the dirty bitmap along with saving it > + * using the QEMUFile pointer. > + */ > + while (b->current_iteration <= b->iterations) { > + if (!runstate_is_running() || > + b->state != LOG_BITMAP_STATE_ACTIVE) { > + goto log_thread_end; > + } > + > + /* > + * Need to calculate the ram pages again as there is a > + * possibility of the change in the memory > + */ > + log_bitmap_lock(); > + current_ram_bitmap_pages = last_ram_offset() >> TARGET_PAGE_BITS; > + if (current_ram_bitmap_pages != prev_ram_bitmap_pages) { > + temp_log_bitmap_array = bitmap_new(current_ram_bitmap_pages); > + if (temp_log_bitmap_array == NULL) { > + b->state = LOG_BITMAP_STATE_ERROR; > + log_bitmap_unlock(); > + goto log_thread_end; > + } > + log_bitmap_dirty_bitmap_sync(temp_log_bitmap_array); > + } else { > + log_bitmap_dirty_bitmap_sync(b->log_bitmap_array); > + } > + log_bitmap_unlock(); > + > + if (current_ram_bitmap_pages != prev_ram_bitmap_pages) { > + prev_ram_bitmap_pages = current_ram_bitmap_pages; > + bitmap_size = BITS_TO_LONGS(current_ram_bitmap_pages) * > + sizeof(unsigned long); > + if (b->log_bitmap_array) { > + g_free(b->log_bitmap_array); > + } > + b->log_bitmap_array = temp_log_bitmap_array; > + temp_log_bitmap_array = NULL; > + if (log_bitmap_ram_block_info_dump(fd, current_ram_bitmap_pages, > + true)) { > + b->state = LOG_BITMAP_STATE_ERROR; > + goto log_thread_end; > + } > + } else { > + if (log_bitmap_ram_block_info_dump(fd, current_ram_bitmap_pages, > + false)) { > + b->state = LOG_BITMAP_STATE_ERROR; > + goto log_thread_end; > + } > + } > + > + ret = qemu_write_full(fd, b->log_bitmap_array, bitmap_size); > + if (ret < bitmap_size) { > + b->state = LOG_BITMAP_STATE_ERROR; > + goto log_thread_end; > + } > + > + ret = qemu_write_full(fd, &marker, sizeof(char)); > + if (ret < sizeof(char)) { > + b->state = LOG_BITMAP_STATE_ERROR; > + goto log_thread_end; > + } > + g_usleep(b->current_period * 1000); > + b->current_iteration++; > + bitmap_zero(b->log_bitmap_array, current_ram_bitmap_pages); > + } > + > + /* > + * stop the logging period. > + */ > + log_thread_end: > + log_bitmap_close(b); > + log_bitmap_update_status(b); > + qemu_process_set(QEMU_PROCESS_NONE); > + return NULL; > +} > + > +void qmp_log_dirty_bitmap(const char *filename, bool has_iterations, > + int64_t iterations, bool has_period, > + int64_t period, Error **errp) > +{ > + int fd = -1; > + BitmapLogState *b = log_bitmap_get_current_state(); > + Error *local_err = NULL; > + > + if (!runstate_is_running()) { > + error_setg(errp, "Guest is not in a running state"); > + return; > + } > + > + if (!qemu_process_check(QEMU_PROCESS_NONE) && > + !qemu_process_check(QEMU_PROCESS_BITMAP_DUMP)) { > + error_setg(errp, "Dirty bitmap dumping not possible, since %s " > + "is in progress.\n", get_qemu_process_as_string()); > + return; > + } > + > + qemu_process_set(QEMU_PROCESS_BITMAP_DUMP); > + > + if (b->state == LOG_BITMAP_STATE_ACTIVE || > + b->state == LOG_BITMAP_STATE_CANCELING) { > + error_setg(errp, "dirty bitmap dump in progress"); > + return; > + } > + > + b->state = LOG_BITMAP_STATE_NONE; > + > + /* > + * checking the iteration range > + */ > + if (!has_iterations) { > + b->iterations = MIN_ITERATION_VALUE; > + } else if (!value_in_range(iterations, MIN_ITERATION_VALUE, > + MAX_ITERATION_VALUE, "iterations", > &local_err)) { > + if (local_err) { > + error_propagate(errp, local_err); > + } > + return; > + } else { > + b->iterations = iterations; > + } > + > + /* > + * checking the period range > + */ > + if (!has_period) { > + b->current_period = MIN_PERIOD_VALUE; > + } else if (!value_in_range(period, MIN_PERIOD_VALUE, > + MAX_PERIOD_VALUE, "period", &local_err)) { > + if (local_err) { > + error_propagate(errp, local_err); > + } > + return; > + } else { > + b->current_period = period; > + } > + > + fd = qemu_open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, > S_IRUSR); > + if (fd < 0) { > + error_setg_file_open(errp, errno, filename); > + return; > + } > + > + b->fd = fd; > + qemu_thread_create(&b->thread, "dirty-bitmap-dump", > + bitmap_logging_thread, b, > + QEMU_THREAD_JOINABLE); > + > + return; > +} > + > void qmp_xen_save_devices_state(const char *filename, Error **errp) > { > QEMUFile *f; > diff --git a/vl.c b/vl.c > index fe451aa..2fa97b3 100644 > --- a/vl.c > +++ b/vl.c > @@ -205,6 +205,8 @@ bool qemu_uuid_set; > static QEMUBootSetHandler *boot_set_handler; > static void *boot_set_opaque; > > +int dirty_bitmap_user; > + > static NotifierList exit_notifiers = > NOTIFIER_LIST_INITIALIZER(exit_notifiers); > > @@ -751,6 +753,27 @@ void vm_start(void) > qapi_event_send_resume(&error_abort); > } > > +/* > + * A global variable to decide which process will only > + * execute migration or bitmap dump > + */ > + > +static QemuProcess dbu = QEMU_PROCESS_NONE; > + > +bool qemu_process_check(QemuProcess user) > +{ > + return user == dbu; > +} > + > +void qemu_process_set(QemuProcess user) > +{ > + dbu = user; > +} > + > +const char *get_qemu_process_as_string(void) > +{ > + return QemuProcess_lookup[dbu]; > +} > > /***********************************************************/ > /* real time host monotonic timer */ > @@ -4518,6 +4541,7 @@ int main(int argc, char **argv, char **envp) > } > > if (incoming) { > + qemu_process_set(QEMU_PROCESS_MIGRATION); > Error *local_err = NULL; > qemu_start_incoming_migration(incoming, &local_err); > if (local_err) { > -- > 1.9.3 > -- Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK