In order to support non-disruptive compactions in the future we will need to use mutliple files for a single ovsdb-log. This will allow the process to continue writing to the log while another part of the log is being compacted at the same time.
Signed-off-by: Felix Huettner <felix.huettner@stackit.cloud> --- ovsdb/log.c | 169 ++++++++++++++++++++++++++++++++++++++++-- ovsdb/log.h | 11 ++- ovsdb/ovsdb-tool.c | 8 +- ovsdb/raft.c | 2 +- tests/ovsdb-log.at | 12 +-- tests/ovsdb-server.at | 27 ++++--- tests/ovsdb-tool.at | 4 +- 7 files changed, 201 insertions(+), 32 deletions(-) diff --git a/ovsdb/log.c b/ovsdb/log.c index f7a21ad36..339ce2739 100644 --- a/ovsdb/log.c +++ b/ovsdb/log.c @@ -37,11 +37,13 @@ #include "seq.h" #include "sha1.h" #include "socket-util.h" +#include "timeval.h" #include "transaction.h" #include "util.h" #define OVSDB_LOG_DATA_KEY "logdata" #define OVSDB_LOG_META_KEY "logmeta" +#define OVSDB_LOG_NEXT_FILE_KEY "next_file" VLOG_DEFINE_THIS_MODULE(ovsdb_log); @@ -74,6 +76,21 @@ enum ovsdb_log_state { OVSDB_LOG_BROKEN, /* Disk on fire, see 'error' for details. */ }; +/* The ovsdb log (if used for writing) consists of up to 3 files that are used + * concurrently. The files may reference the following files. + * + * base: This file contains the state after a snapshot (or initial database + * creation). It is never written to, but will be replaced during + * ovsdb_log_replace_*. + * May be NULL during the initial read of the log. + * curr: The currently active log that is being written to. May contain any + * number of commited data. New entries are written to this file only. + * This is never NULL. + * old: During compaction curr is replaced with a new file and the previous + * curr is stored as old. The compcation will merge base and old to a + * single consistent state and afterwards replace base with it. + * Unless a compaction is currently running this is NULL. + */ struct ovsdb_log_file { off_t prev_offset; /* While reading the offset before the latest @@ -94,9 +111,13 @@ struct ovsdb_log { enum ovsdb_log_open_mode open_mode; bool may_lock; + bool only_single_file; /* To be used for temporary logs (e.g. during + * snapshots). */ char *display_name; /* For use in log messages, etc. */ char *magic; + struct ovsdb_log_file *base; + struct ovsdb_log_file *old; struct ovsdb_log_file *curr; }; @@ -153,6 +174,7 @@ ovsdb_log_file_open(const char *name, break; case OVSDB_LOG_CREATE_EXCL: + case OVSDB_LOG_CREATE_EXCL_SINGLE: #ifndef _WIN32 if (stat(name, &s) == -1 && errno == ENOENT && lstat(name, &s) == 0 && S_ISLNK(s.st_mode)) { @@ -187,7 +209,9 @@ ovsdb_log_file_open(const char *name, fd = open(name, flags, 0660); } if (fd < 0) { - const char *op = (open_mode == OVSDB_LOG_CREATE_EXCL ? "create" + const char *op = ( + (open_mode == OVSDB_LOG_CREATE_EXCL || + open_mode == OVSDB_LOG_CREATE_EXCL_SINGLE) ? "create" : open_mode == OVSDB_LOG_CREATE ? "create or open" : "open"); error = ovsdb_io_error(errno, "%s: %s failed", name, op); @@ -291,6 +315,7 @@ ovsdb_log_open(const char *name, const char *magic, /* If we can create a new file, we need to know what kind of magic to * use, so there must be only one kind. */ if (open_mode == OVSDB_LOG_CREATE_EXCL || + open_mode == OVSDB_LOG_CREATE_EXCL_SINGLE || open_mode == OVSDB_LOG_CREATE) { ovs_assert(!strchr(magic, '|')); } @@ -325,6 +350,7 @@ ovsdb_log_open(const char *name, const char *magic, log->magic = actual_magic; log->open_mode = open_mode; log->may_lock = may_lock; + log->only_single_file = open_mode == OVSDB_LOG_CREATE_EXCL_SINGLE; log->curr = file; *filep = log; @@ -374,6 +400,8 @@ ovsdb_log_close(struct ovsdb_log *file) ovsdb_error_destroy(file->error); free(file->display_name); free(file->magic); + ovsdb_log_file_close(file->base); + ovsdb_log_file_close(file->old); ovsdb_log_file_close(file->curr); free(file); } @@ -607,7 +635,43 @@ ovsdb_log_read(struct ovsdb_log *log, struct json **jsonp) *jsonp = data; break; } else { - OVS_NOT_REACHED(); + const struct json *next_file = shash_find_data( + json_object(metadata), OVSDB_LOG_NEXT_FILE_KEY); + if (next_file) { + const char *next_filename = json_string(next_file); + char *abs_name = follow_symlinks(log->curr->name); + char *basedir = dir_name(abs_name); + char *filename = xasprintf("%s/%s", basedir, + next_filename); + struct ovsdb_log_file *next; + error = ovsdb_log_file_open( + filename, log->magic, log->open_mode, + log->may_lock, &next, NULL); + free(abs_name); + free(basedir); + free(filename); + if (error) { + json_destroy(*jsonp); + *jsonp = NULL; + return error; + } + if (!log->base) { + /* We just finished reading the base file */ + log->base = log->curr; + log->curr = next; + } else { + /* We already have a base file, so we must be reading from + * a db that is in a transient state of a compaction. + * For now this needs to be fixed manually. */ + log->state = OVSDB_LOG_BROKEN; + json_destroy(*jsonp); + *jsonp = NULL; + return ovsdb_error(NULL, "%s: can not further follow " + "referenced data files", next_filename); + } + } + json_destroy(*jsonp); + *jsonp = NULL; } } @@ -627,6 +691,9 @@ void ovsdb_log_unread(struct ovsdb_log *file) { ovs_assert(file->state == OVSDB_LOG_READ); + + ovs_assert(file->curr->prev_offset > 0 || !file->base); + file->curr->offset = file->curr->prev_offset; } @@ -656,11 +723,17 @@ ovsdb_log_truncate(struct ovsdb_log *file) /* Removes all the data from the log by moving current offset to zero and * truncating the file to zero bytes. After this operation the file is empty - * and in a write state. */ + * and in a write state. + * This is currently only supported in combination with + * OVSDB_LOG_CREATE_EXCL_SINGLE. */ struct ovsdb_error * OVS_WARN_UNUSED_RESULT ovsdb_log_reset(struct ovsdb_log *file) { + ovs_assert(!file->base && !file->old && + file->open_mode == OVSDB_LOG_CREATE_EXCL_SINGLE); + ovsdb_error_destroy(file->error); + file->curr->offset = file->curr->prev_offset = 0; file->error = ovsdb_log_truncate(file); if (file->error) { @@ -734,6 +807,35 @@ ovsdb_log_write_file(struct ovsdb_log *log, struct ovsdb_log_file *file, return NULL; } +static struct ovsdb_error * OVS_WARN_UNUSED_RESULT +ovsdb_log_write_next_file(struct ovsdb_log *log, struct ovsdb_log_file *file, + const char *filepath) +{ + char *filename = base_name(filepath); + struct json *inner = json_object_create(); + json_object_put_string(inner, OVSDB_LOG_NEXT_FILE_KEY, filename); + free(filename); + struct json *json = json_object_create(); + json_object_put(json, OVSDB_LOG_META_KEY, inner); + + struct ovsdb_error *error = ovsdb_log_write_file(log, file, json); + json_destroy(json); + return error; +} + +static char * +ovsdb_log_get_new_filename(struct ovsdb_log *log) +{ + struct ovsdb_log_file *file = log->curr; + if (log->base) { + file = log->base; + } + char *deref_name = follow_symlinks(file->name); + char *new_curr_name = xasprintf("%s.%lld", deref_name, time_wall_msec()); + free(deref_name); + return new_curr_name; +} + static struct ovsdb_error * ovsdb_log_write_(struct ovsdb_log *log, const struct json *json) { @@ -762,6 +864,43 @@ ovsdb_log_write_(struct ovsdb_log *log, const struct json *json) return ovsdb_error_clone(log->error); } + if (!log->base && !log->only_single_file) { + /* When we start writing to a log we will always ensure we + * have a base file for later snapshoting. */ + struct ovsdb_log_file *new_curr; + char *new_curr_name = ovsdb_log_get_new_filename(log); + + enum ovsdb_log_open_mode mode = log->open_mode; + if (mode == OVSDB_LOG_READ_WRITE) { + mode = OVSDB_LOG_CREATE; + } + + log->error = ovsdb_log_file_open(new_curr_name, log->magic, + mode, log->may_lock, + &new_curr, NULL); + if (log->error) { + log->state = OVSDB_LOG_WRITE_ERROR; + free(new_curr_name); + return ovsdb_error_clone(log->error); + } + + log->error = ovsdb_log_write_next_file(log, log->curr, new_curr_name); + if (log->error) { + /* Close and remove the newly created file. */ + ovsdb_log_file_close(new_curr); + unlink(new_curr_name); + free(new_curr_name); + log->state = OVSDB_LOG_WRITE_ERROR; + return ovsdb_error_clone(log->error); + } + + free(new_curr_name); + log->base = log->curr; + log->curr = new_curr; + } + + ovs_assert(log->base || log->only_single_file); + if (!json) { return NULL; } @@ -900,6 +1039,7 @@ struct ovsdb_error * OVS_WARN_UNUSED_RESULT ovsdb_log_replace_start(struct ovsdb_log *old, struct ovsdb_log **newp) { + ovs_assert(!old->old); ovs_assert(old->curr->lockfile); /* If old->name is a symlink, then we want the new file to be in the same @@ -919,7 +1059,7 @@ ovsdb_log_replace_start(struct ovsdb_log *old, } /* Create temporary file. */ - error = ovsdb_log_open(tmp_name, old->magic, OVSDB_LOG_CREATE_EXCL, + error = ovsdb_log_open(tmp_name, old->magic, OVSDB_LOG_CREATE_EXCL_SINGLE, false, newp); free(tmp_name); return error; @@ -928,6 +1068,9 @@ ovsdb_log_replace_start(struct ovsdb_log *old, struct ovsdb_error * OVS_WARN_UNUSED_RESULT ovsdb_log_replace_commit(struct ovsdb_log *old, struct ovsdb_log *new) { + /* We may not replace and compact at the same time */ + ovs_assert(!old->old); + struct ovsdb_error *error = ovsdb_log_commit_block(new); if (error) { ovsdb_log_replace_abort(new); @@ -953,6 +1096,8 @@ ovsdb_log_replace_commit(struct ovsdb_log *old, struct ovsdb_log *new) * to test both strategies on Unix-like systems, and to make the code * easier to read. */ if (!rename_open_files) { + fclose(old->base->stream); + old->base->stream = NULL; fclose(old->curr->stream); old->curr->stream = NULL; @@ -963,13 +1108,27 @@ ovsdb_log_replace_commit(struct ovsdb_log *old, struct ovsdb_log *new) /* Rename 'old' to 'new'. We dereference the old name because, if it is a * symlink, we want to replace the referent of the symlink instead of the * symlink itself. */ - char *deref_name = follow_symlinks(old->curr->name); + char *deref_name = follow_symlinks(old->base->name); error = ovsdb_rename(new->curr->name, deref_name); free(deref_name); if (error) { ovsdb_log_replace_abort(new); return error; } + + /* By now we are safely in the state where 'new' is new correct state. + * The new 'cur' will now replace the old 'cur'. The old 'base' will be + * closed. */ + if (old->base) { + char *old_curr_file = old->curr->name; + old->curr->name = NULL; + ovsdb_log_file_close(old->curr); + unlink(old_curr_file); + free(old_curr_file); + old->curr = old->base; + old->base = NULL; + } + if (rename_open_files) { fsync_parent_dir(old->curr->name); old->curr->stream = new->curr->stream; diff --git a/ovsdb/log.h b/ovsdb/log.h index b1db234b0..6161c0d23 100644 --- a/ovsdb/log.h +++ b/ovsdb/log.h @@ -45,10 +45,13 @@ struct ovsdb_log; /* Access mode for opening an OVSDB log. */ enum ovsdb_log_open_mode { - OVSDB_LOG_READ_ONLY, /* Open existing file, read-only. */ - OVSDB_LOG_READ_WRITE, /* Open existing file, read/write. */ - OVSDB_LOG_CREATE_EXCL, /* Create new file, read/write. */ - OVSDB_LOG_CREATE /* Create or open file, read/write. */ + OVSDB_LOG_READ_ONLY, /* Open existing file, read-only. */ + OVSDB_LOG_READ_WRITE, /* Open existing file, read/write. */ + OVSDB_LOG_CREATE_EXCL, /* Create new file, read/write. */ + OVSDB_LOG_CREATE_EXCL_SINGLE, /* Create new file, read/write. Do not create + * multiple files for writing. Only use this + * for short lived files. */ + OVSDB_LOG_CREATE /* Create or open file, read/write. */ }; /* 'magic' for use with ovsdb_log_open() for OVSDB databases (see ovsdb(5)). */ diff --git a/ovsdb/ovsdb-tool.c b/ovsdb/ovsdb-tool.c index 5ab7988e9..e62fb54b3 100644 --- a/ovsdb/ovsdb-tool.c +++ b/ovsdb/ovsdb-tool.c @@ -278,7 +278,8 @@ do_create(struct ovs_cmdl_context *ctx) /* Create database file. */ check_ovsdb_error(ovsdb_log_open(db_file_name, OVSDB_MAGIC, - OVSDB_LOG_CREATE_EXCL, true, &log)); + OVSDB_LOG_CREATE_EXCL_SINGLE, true, + &log)); check_ovsdb_error(ovsdb_log_write_and_free(log, json)); check_ovsdb_error(ovsdb_log_commit_block(log)); ovsdb_log_close(log); @@ -352,7 +353,8 @@ write_standalone_db(const char *file_name, const char *comment, { struct ovsdb_log *log; struct ovsdb_error *error = ovsdb_log_open(file_name, OVSDB_MAGIC, - OVSDB_LOG_CREATE, false, &log); + OVSDB_LOG_CREATE_EXCL_SINGLE, + false, &log); if (error) { return error; } @@ -1708,7 +1710,7 @@ do_cluster_standalone(struct ovs_cmdl_context *ctx) OVSDB_MAGIC"|"RAFT_MAGIC, OVSDB_LOG_READ_ONLY, true, &log)); check_ovsdb_error(ovsdb_log_open(db_file_name, OVSDB_MAGIC, - OVSDB_LOG_CREATE_EXCL, true, + OVSDB_LOG_CREATE_EXCL_SINGLE, true, &db_log_data)); if (strcmp(ovsdb_log_get_magic(log), RAFT_MAGIC) != 0) { ovs_fatal(0, "Database is not clustered db.\n"); diff --git a/ovsdb/raft.c b/ovsdb/raft.c index e2b3eb60e..cf9285b7e 100644 --- a/ovsdb/raft.c +++ b/ovsdb/raft.c @@ -502,7 +502,7 @@ raft_create_cluster(const char *file_name, const char *name, /* Create log file. */ struct ovsdb_log *log; - error = ovsdb_log_open(file_name, RAFT_MAGIC, OVSDB_LOG_CREATE_EXCL, + error = ovsdb_log_open(file_name, RAFT_MAGIC, OVSDB_LOG_CREATE_EXCL_SINGLE, true, &log); if (error) { return error; diff --git a/tests/ovsdb-log.at b/tests/ovsdb-log.at index a73990a15..d50fbbf4d 100644 --- a/tests/ovsdb-log.at +++ b/tests/ovsdb-log.at @@ -243,7 +243,7 @@ file: write:{"x":0} successful file: write:{"x":1} successful file: write:{"x":2} successful ]], [ignore]) -AT_CHECK([echo 'xxx' >> file]) +AT_CHECK([bash -c "echo 'xxx' >> file.*"]) AT_CHECK( [test-ovsdb log-io file read-only read read read read | sed -e 's|error: .*file[[.0-9]]*:|error: file:|'], [0], [[file: open successful @@ -265,7 +265,7 @@ file: write:{"x":0} successful file: write:{"x":1} successful file: write:{"x":2} successful ]], [ignore]) -AT_CHECK([echo 'xxx' >> file]) +AT_CHECK([bash -c "echo 'xxx' >> file.*"]) AT_CHECK( [[test-ovsdb log-io file read/write read read read read 'write:{"x":3}' | sed -e 's|error: .*file[.0-9]*:|error: file:|']], [0], [[file: open successful @@ -297,8 +297,9 @@ file: write:{"x":0} successful file: write:{"x":1} successful file: write:{"x":2} successful ]], [ignore]) -AT_CHECK([[sed 's/{"x":2}/{"x":3}/' < file > file.tmp]]) +AT_CHECK([[bash -c "sed 's/{\"x\":2}/{\"x\":3}/' < file.* > file.tmp"]]) AT_CHECK([mv file.tmp file]) +AT_CHECK([[bash -c "rm file.*"]]) AT_CHECK([[grep -c '{"x":3}' file]], [0], [1 ]) AT_CHECK( @@ -330,8 +331,9 @@ file: write:{"x":0} successful file: write:{"x":1} successful file: write:{"x":2} successful ]], [ignore]) -AT_CHECK([[sed 's/{"x":2}/2/' < file > file.tmp]]) +AT_CHECK([[bash -c "sed 's/{\"x\":2}/2/' < file.* > file.tmp"]]) AT_CHECK([mv file.tmp file]) +AT_CHECK([[bash -c "rm file.*"]]) AT_CHECK([[grep -c '^{"logdata":2}$' file]], [0], [1 ]) AT_CHECK( @@ -363,7 +365,7 @@ file: write:{"x":0} successful file: write:{"x":1} successful file: write:{"x":2} successful ]], [ignore]) -AT_CHECK([[printf '%s\n%s\n' 'OVSDB JSON 5 d910b02871075d3156ec8675dfc95b7d5d640aa6' 'null' >> file]]) +AT_CHECK([[bash -c "printf '%s\n%s\n' 'OVSDB JSON 5 d910b02871075d3156ec8675dfc95b7d5d640aa6' 'null' >> file.*"]]) AT_CHECK( [[test-ovsdb log-io file read/write read read read read 'write:{"replacement data":0}' | sed -e 's|error: .*file[.0-9]*:|error: file:|']], [0], [[file: open successful diff --git a/tests/ovsdb-server.at b/tests/ovsdb-server.at index ef27d298e..57322ccb0 100644 --- a/tests/ovsdb-server.at +++ b/tests/ovsdb-server.at @@ -75,7 +75,7 @@ AT_CHECK([ovsdb-server --remote=punix:socket db --run="sh txnfile"], [0], [stdou cat stdout >> output dnl Add some crap to the database log and run another transaction, which should dnl ignore the crap and truncate it out of the log. -echo 'xxx' >> db +bash -c "echo 'xxx' >> db.*" AT_DATA([txnfile], [[ovsdb-client transact unix:socket \ '["ordinals", {"op": "insert", @@ -122,7 +122,7 @@ cat stdout >> output dnl Add some crap to the database log and run another transaction, which should dnl ignore the crap and truncate it out of the log. bash -c "echo 'OVSDB JSON 26 d1cd8c1ecd3aff070019a75973f35832ff0cfe15 -{\"logdata\":{\"invalid\":{}}}' >> db" +{\"logdata\":{\"invalid\":{}}}' >> db.*" AT_DATA([txnfile], [[ovsdb-client transact unix:socket \ '["ordinals", {"op": "insert", @@ -1100,7 +1100,7 @@ ovsdb_check_online_compaction() { [0], [stdout]) if test $model = standalone; then dnl Check that all the crap is in fact in the database log. - AT_CHECK([[uuidfilt db | grep -v ^OVSDB | \ + AT_CHECK([[uuidfilt dir/db* | grep -v ^OVSDB | grep -v logmeta | \ sed 's/"_date":[0-9]*/"_date":0/' | sed 's/"_is_diff":true,//' | \ ovstest test-json --multiple -]], [0], [[{"logdata":{"cksum":"12345678 9","name":"ordinals","tables":{"ordinals":{"columns":{"name":{"type":"string"},"number":{"type":"integer"}},"indexes":[["number"],["number","name"]]}},"version":"5.1.3"}} @@ -1125,7 +1125,7 @@ ovsdb_check_online_compaction() { ]]) else dnl Check that at least there's a lot of transactions. - AT_CHECK([test `wc -l < db` -gt 50]) + AT_CHECK([test `cat dir/db* | wc -l` -gt 50]) fi dnl Dump out and check the actual database contents. AT_CHECK([ovsdb-client dump unix:socket ordinals], [0], [stdout]) @@ -1162,7 +1162,7 @@ ovs-appctl: ovsdb-server: server returned an error # order of the records is not predictable, but there should only be 4 lines # in it now in the standalone case AT_CAPTURE_FILE([db]) - compacted_lines=`wc -l < db` + compacted_lines=`cat dir/db* | wc -l` echo compacted_lines=$compacted_lines if test $model = standalone; then AT_CHECK([test $compacted_lines -eq 4]) @@ -1191,12 +1191,12 @@ _uuid name number [0], [[[{"count":3}] ]], [ignore]) - dnl There should be 6 lines in the log now, for the standalone case, + dnl There should be 8 lines in the log now, for the standalone case, dnl and for the clustered case the file should at least have grown. - updated_lines=`wc -l < db` + updated_lines=`cat dir/db* | wc -l` echo compacted_lines=$compacted_lines updated_lines=$updated_lines if test $model = standalone; then - AT_CHECK([test $updated_lines -eq 6]) + AT_CHECK([test $updated_lines -eq 8]) else AT_CHECK([test $updated_lines -gt $compacted_lines]) fi @@ -2810,12 +2810,12 @@ dnl Make a copy of a database for later replay. AT_CHECK([cp db ./replay_dir/db.copy]) dnl Starting a dummy server only to reserve some tcp port. -AT_CHECK([cp db db.tmp]) +AT_CHECK([cp db port_reserve_db.tmp]) AT_CHECK([ovsdb-server -vfile -vvlog:off --log-file=listener.log dnl --detach --no-chdir dnl --pidfile=2.pid --unixctl=unixctl2 dnl --remote=ptcp:0:127.0.0.1 dnl - db.tmp], [0], [stdout], [stderr]) + port_reserve_db.tmp], [0], [stdout], [stderr]) PARSE_LISTENING_PORT([listener.log], [BAD_TCP_PORT]) dnl Start ovsdb-server with recording enabled. @@ -2920,9 +2920,12 @@ OVS_WAIT_WHILE([test -e ovsdb-server.pid]) dnl Stripping out timestamps from database files. Also clearing record dnl hashes in database files, since dates inside are different. +dnl Removing any logmeta records since the referenced filename will be +dnl different due to timestamps and also have a different length. m4_define([CLEAN_DB_FILE], - [sed 's/\(OVSDB JSON [[0-9]]*\).*$/\1/g' $1 | dnl - sed 's/"_date":[[0-9]]*/"_date":<clared>/g' > $2]) + [sed 's/\(OVSDB JSON [[0-9]]*\).*$/\1/g' $1* | dnl + sed 's/"_date":[[0-9]]*/"_date":<clared>/g' | dnl + sed -s 'N;/logmeta/!P;D' > $2]) CLEAN_DB_FILE([db], [db.clear]) CLEAN_DB_FILE([./replay_dir/db.copy], [./replay_dir/db.copy.clear]) diff --git a/tests/ovsdb-tool.at b/tests/ovsdb-tool.at index 693524017..4c377057a 100644 --- a/tests/ovsdb-tool.at +++ b/tests/ovsdb-tool.at @@ -43,7 +43,7 @@ AT_CHECK([[ovsdb-tool transact db ' AT_CHECK([uuidfilt stdout], [0], [[[{"uuid":["uuid","<0>"]},{}] ]]) -AT_CHECK([grep "add row for 5" db], [0], [ignore]) +AT_CHECK([bash -c "grep 'add row for 5' db*"], [0], [ignore]) AT_CLEANUP AT_SETUP([ovsdb-tool compact]) @@ -93,7 +93,7 @@ AT_CHECK( done]], [0], [stdout], [ignore]) dnl Check that all the crap is in fact in the database log. -AT_CHECK([[uuidfilt db | grep -v ^OVSDB | sed 's/"_date":[0-9]*/"_date":0/' | \ +AT_CHECK([[uuidfilt dir/db* | grep -v ^OVSDB | grep -v "logmeta" | sed 's/"_date":[0-9]*/"_date":0/' | \ sed 's/"_is_diff":true,//' | ovstest test-json --multiple -]], [0], [[{"logdata":{"cksum":"12345678 9","name":"ordinals","tables":{"ordinals":{"columns":{"name":{"type":"string"},"number":{"type":"integer"}},"indexes":[["number"],["number","name"]]}},"version":"5.1.3"}} {"logdata":{"_comment":"add row for zero 0","_date":0,"ordinals":{"<0>":{"name":"zero"}}}} -- 2.43.0 _______________________________________________ dev mailing list d...@openvswitch.org https://mail.openvswitch.org/mailman/listinfo/ovs-dev