On Linux jsonrpc server now users poll group by default. It can
be disabled by using an undocumented --disable-epoll command line
options.
For ovsdb-server to maintain 1000 idle connections over TCP with the
default 5s probe interval, the CPU load dropped from 48% to 14%.
Signed-off-by: Andy Zhou <az...@ovn.org>
---
NEWS | 5 +++-
ovsdb/jsonrpc-server.c | 75 ++++++++++++++++++++++++++++++++++++++++++++++----
ovsdb/jsonrpc-server.h | 2 +-
ovsdb/ovsdb-server.c | 43 +++++++++++++++++++++--------
tests/ovsdb-server.at | 11 ++++++++
5 files changed, 117 insertions(+), 19 deletions(-)
diff --git a/NEWS b/NEWS
index 18fca10..e5eeba4 100644
--- a/NEWS
+++ b/NEWS
@@ -17,7 +17,10 @@ Post-v2.5.0
- ovsdb-server:
* Remove max number of sessions limit, to enable connection scaling
testing.
-
+ - ovsdb-server:
+ * On Linux, ovsdb server uses poll group, which uses epoll(7) to
+ improve connection scaling. Add an undocumented --disable-epoll
+ switch for testing.
v2.5.0 - xx xxx xxxx
---------------------
diff --git a/ovsdb/jsonrpc-server.c b/ovsdb/jsonrpc-server.c
index 0d23b77..d2f16c8 100644
--- a/ovsdb/jsonrpc-server.c
+++ b/ovsdb/jsonrpc-server.c
@@ -28,6 +28,7 @@
#include "ovsdb-error.h"
#include "ovsdb-parser.h"
#include "ovsdb.h"
+#include "poll-group.h"
#include "poll-loop.h"
#include "reconnect.h"
#include "row.h"
@@ -103,8 +104,11 @@ struct ovsdb_jsonrpc_server {
struct ovsdb_server up;
unsigned int n_sessions;
struct shash remotes; /* Contains "struct ovsdb_jsonrpc_remote *"s. */
+ struct poll_group *poll_group; /* group poll jsonrpc sessions. */
};
+static void ovsdb_jsonrpc_server_run_poll_group(struct poll_group *);
+
/* A configured remote. This is either a passive stream listener plus a list
* of the currently connected sessions, or a list of exactly one active
* session. */
@@ -124,13 +128,24 @@ static void ovsdb_jsonrpc_server_del_remote(struct
shash_node *);
/* Creates and returns a new server to provide JSON-RPC access to an OVSDB.
*
* The caller must call ovsdb_jsonrpc_server_add_db() for each database to
- * which 'server' should provide access. */
+ * which 'server' should provide access.
+ *
+ * On platforms where epoll() is supported, using epoll() can be more
+ * efficient than using poll(), and it will be enabled by default.
+ * To disable it, set 'disable_epoll to false.
+ */
struct ovsdb_jsonrpc_server *
-ovsdb_jsonrpc_server_create(void)
+ovsdb_jsonrpc_server_create(bool disable_epoll)
{
struct ovsdb_jsonrpc_server *server = xzalloc(sizeof *server);
+ struct poll_group *poll_group = NULL;
+
ovsdb_server_init(&server->up);
shash_init(&server->remotes);
+ if (!disable_epoll) {
+ poll_group = poll_group_create("jsonrpc-sessions-pg");
+ }
+ server->poll_group = poll_group;
return server;
}
@@ -318,6 +333,7 @@ void
ovsdb_jsonrpc_server_run(struct ovsdb_jsonrpc_server *svr)
{
struct shash_node *node;
+ struct poll_group *poll_group = svr->poll_group;
SHASH_FOR_EACH (node, &svr->remotes) {
struct ovsdb_jsonrpc_remote *remote = node->data;
@@ -329,10 +345,13 @@ ovsdb_jsonrpc_server_run(struct ovsdb_jsonrpc_server *svr)
error = pstream_accept(remote->listener, &stream);
if (!error) {
struct jsonrpc_session *js;
+ struct ovsdb_jsonrpc_session *ovsdb_js;
js = jsonrpc_session_open_unreliably(jsonrpc_open(stream),
remote->dscp);
- ovsdb_jsonrpc_session_create(remote, js);
- } else if (error != EAGAIN) {
+ ovsdb_js = ovsdb_jsonrpc_session_create(remote, js);
+ stream_join(stream, svr->poll_group, ovsdb_js);
+ }
+ else if (error != EAGAIN) {
VLOG_WARN_RL(&rl, "%s: accept failed: %s",
pstream_get_name(remote->listener),
ovs_strerror(error));
@@ -341,11 +360,16 @@ ovsdb_jsonrpc_server_run(struct ovsdb_jsonrpc_server *svr)
ovsdb_jsonrpc_session_run_all(remote);
}
+
+ if (poll_group) {
+ ovsdb_jsonrpc_server_run_poll_group(poll_group);
+ }
}
void
ovsdb_jsonrpc_server_wait(struct ovsdb_jsonrpc_server *svr)
{
+ struct poll_group *poll_group = svr->poll_group;
struct shash_node *node;
SHASH_FOR_EACH (node, &svr->remotes) {
@@ -357,6 +381,7 @@ ovsdb_jsonrpc_server_wait(struct ovsdb_jsonrpc_server *svr)
ovsdb_jsonrpc_session_wait_all(remote);
}
+ poll_group_poll_wait(poll_group);
}
/* Adds some memory usage statistics for 'svr' into 'usage', for use with
@@ -491,6 +516,27 @@ ovsdb_jsonrpc_session_run_all(struct ovsdb_jsonrpc_remote
*remote)
struct ovsdb_jsonrpc_session *s, *next;
LIST_FOR_EACH_SAFE (s, next, node, &remote->sessions) {
+ if (poll_block_waken_by_timer() ||
+ !jsonrpc_session_joined_poll_group(s->js)) {
+ int error = ovsdb_jsonrpc_session_run(s);
+ if (error) {
+ ovsdb_jsonrpc_session_close(s);
+ }
+ }
+ }
+}
+
+static void
+ovsdb_jsonrpc_server_run_poll_group(struct poll_group *group)
+{
+ void **sessions;
+ size_t n, i;
+
+ poll_group_get_events(group, &sessions, &n);
+
+ for (i = 0; i < n; i++) {
+ struct ovsdb_jsonrpc_session *s;
+ s = sessions[i];
int error = ovsdb_jsonrpc_session_run(s);
if (error) {
ovsdb_jsonrpc_session_close(s);
@@ -499,9 +545,19 @@ ovsdb_jsonrpc_session_run_all(struct ovsdb_jsonrpc_remote
*remote)
}
static void
+ovsdb_jsonrpc_session_poll_group_update(struct ovsdb_jsonrpc_session *s)
+{
+ if (ovsdb_jsonrpc_monitor_needs_flush(s) ||
+ jsonrpc_session_get_backlog(s->js) ||
+ jsonrpc_session_has_pending_input(s->js)) {
+
+ jsonrpc_session_poll_group_update(s->js, true);
+ }
+}
+
+static void
ovsdb_jsonrpc_session_wait(struct ovsdb_jsonrpc_session *s)
{
- jsonrpc_session_wait(s->js);
if (!jsonrpc_session_get_backlog(s->js)) {
if (ovsdb_jsonrpc_monitor_needs_flush(s)) {
poll_immediate_wake();
@@ -517,7 +573,14 @@ ovsdb_jsonrpc_session_wait_all(struct ovsdb_jsonrpc_remote
*remote)
struct ovsdb_jsonrpc_session *s;
LIST_FOR_EACH (s, node, &remote->sessions) {
- ovsdb_jsonrpc_session_wait(s);
+ jsonrpc_session_wait(s->js);
+
+ bool update_pg = jsonrpc_session_joined_poll_group(s->js);
+ if (update_pg) {
+ ovsdb_jsonrpc_session_poll_group_update(s);
+ } else {
+ ovsdb_jsonrpc_session_wait(s);
+ }
}
}
diff --git a/ovsdb/jsonrpc-server.h b/ovsdb/jsonrpc-server.h
index 7a0bd31..76c718f 100644
--- a/ovsdb/jsonrpc-server.h
+++ b/ovsdb/jsonrpc-server.h
@@ -23,7 +23,7 @@ struct ovsdb;
struct shash;
struct simap;
-struct ovsdb_jsonrpc_server *ovsdb_jsonrpc_server_create(void);
+struct ovsdb_jsonrpc_server *ovsdb_jsonrpc_server_create(bool);
bool ovsdb_jsonrpc_server_add_db(struct ovsdb_jsonrpc_server *,
struct ovsdb *);
bool ovsdb_jsonrpc_server_remove_db(struct ovsdb_jsonrpc_server *,
diff --git a/ovsdb/ovsdb-server.c b/ovsdb/ovsdb-server.c
index fa662b1..6e21c68 100644
--- a/ovsdb/ovsdb-server.c
+++ b/ovsdb/ovsdb-server.c
@@ -87,6 +87,7 @@ struct server_config {
struct shash *all_dbs;
FILE *config_tmpfile;
struct ovsdb_jsonrpc_server *jsonrpc;
+ bool disable_epoll;
};
static unixctl_cb_func ovsdb_server_add_remote;
static unixctl_cb_func ovsdb_server_remove_remote;
@@ -101,7 +102,7 @@ static void close_db(struct db *db);
static void parse_options(int *argc, char **argvp[],
struct sset *remotes, char **unixctl_pathp,
- char **run_command);
+ char **run_command, bool *disable_epoll);
OVS_NO_RETURN static void usage(void);
static char *reconfigure_remotes(struct ovsdb_jsonrpc_server *,
@@ -115,10 +116,11 @@ static void update_remote_status(const struct
ovsdb_jsonrpc_server *jsonrpc,
struct shash *all_dbs);
static void save_config__(FILE *config_file, const struct sset *remotes,
- const struct sset *db_filenames);
+ const struct sset *db_filenames,
+ const bool disable_epoll);
static void save_config(struct server_config *);
static void load_config(FILE *config_file, struct sset *remotes,
- struct sset *db_filenames);
+ struct sset *db_filenames, bool *disable_epoll);
static void
main_loop(struct ovsdb_jsonrpc_server *jsonrpc, struct shash *all_dbs,
@@ -203,6 +205,7 @@ main(int argc, char *argv[])
{
char *unixctl_path = NULL;
char *run_command = NULL;
+ bool disable_epoll = false; /* Use epoll() by default. */
struct unixctl_server *unixctl;
struct ovsdb_jsonrpc_server *jsonrpc;
struct sset remotes, db_filenames;
@@ -223,7 +226,8 @@ main(int argc, char *argv[])
fatal_ignore_sigpipe();
process_init();
- parse_options(&argc, &argv, &remotes, &unixctl_path, &run_command);
+ parse_options(&argc, &argv, &remotes, &unixctl_path, &run_command,
+ &disable_epoll);
daemon_become_new_user(false);
/* Create and initialize 'config_tmpfile' as a temporary file to hold
@@ -247,16 +251,17 @@ main(int argc, char *argv[])
free(default_db);
}
+ server_config.disable_epoll = disable_epoll;
server_config.remotes = &remotes;
server_config.config_tmpfile = config_tmpfile;
- save_config__(config_tmpfile, &remotes, &db_filenames);
+ save_config__(config_tmpfile, &remotes, &db_filenames, disable_epoll);
daemonize_start(false);
/* Load the saved config. */
- load_config(config_tmpfile, &remotes, &db_filenames);
- jsonrpc = ovsdb_jsonrpc_server_create();
+ load_config(config_tmpfile, &remotes, &db_filenames, &disable_epoll);
+ jsonrpc = ovsdb_jsonrpc_server_create(disable_epoll);
shash_init(&all_dbs);
server_config.all_dbs = &all_dbs;
@@ -1270,12 +1275,14 @@ ovsdb_server_list_databases(struct unixctl_conn *conn,
int argc OVS_UNUSED,
static void
parse_options(int *argcp, char **argvp[],
- struct sset *remotes, char **unixctl_pathp, char **run_command)
+ struct sset *remotes, char **unixctl_pathp, char **run_command,
+ bool *disable_epoll)
{
enum {
OPT_REMOTE = UCHAR_MAX + 1,
OPT_UNIXCTL,
OPT_RUN,
+ OPT_DISABLE_EPOLL,
OPT_BOOTSTRAP_CA_CERT,
OPT_PEER_CA_CERT,
VLOG_OPTION_ENUMS,
@@ -1284,6 +1291,7 @@ parse_options(int *argcp, char **argvp[],
static const struct option long_options[] = {
{"remote", required_argument, NULL, OPT_REMOTE},
{"unixctl", required_argument, NULL, OPT_UNIXCTL},
+ {"disable-epoll", no_argument, NULL, OPT_DISABLE_EPOLL},
#ifndef _WIN32
{"run", required_argument, NULL, OPT_RUN},
#endif
@@ -1301,6 +1309,7 @@ parse_options(int *argcp, char **argvp[],
char *short_options = ovs_cmdl_long_options_to_short_options(long_options);
int argc = *argcp;
char **argv = *argvp;
+ bool disable_epoll_ = false;
sset_init(remotes);
for (;;) {
@@ -1324,6 +1333,10 @@ parse_options(int *argcp, char **argvp[],
*run_command = optarg;
break;
+ case OPT_DISABLE_EPOLL:
+ disable_epoll_ = true;
+ break;
+
case 'h':
usage();
@@ -1363,6 +1376,8 @@ parse_options(int *argcp, char **argvp[],
abort();
}
}
+
+ *disable_epoll = disable_epoll_;
free(short_options);
*argcp -= optind;
@@ -1407,7 +1422,7 @@ sset_to_json(const struct sset *sset)
* 'remotes' and 'db_filenames'. */
static void
save_config__(FILE *config_file, const struct sset *remotes,
- const struct sset *db_filenames)
+ const struct sset *db_filenames, bool disable_epoll)
{
struct json *obj;
char *s;
@@ -1420,6 +1435,7 @@ save_config__(FILE *config_file, const struct sset
*remotes,
obj = json_object_create();
json_object_put(obj, "remotes", sset_to_json(remotes));
json_object_put(obj, "db_filenames", sset_to_json(db_filenames));
+ json_object_put(obj, "disable_epoll", json_boolean_create(disable_epoll));
s = json_to_string(obj, 0);
json_destroy(obj);
@@ -1445,7 +1461,8 @@ save_config(struct server_config *config)
sset_add(&db_filenames, db->filename);
}
- save_config__(config->config_tmpfile, config->remotes, &db_filenames);
+ save_config__(config->config_tmpfile, config->remotes, &db_filenames,
+ config->disable_epoll);
sset_destroy(&db_filenames);
}
@@ -1467,7 +1484,8 @@ sset_from_json(struct sset *sset, const struct json
*array)
/* Clears and replaces 'remotes' and 'dbnames' by a configuration read from
* 'config_file', which must have been previously written by save_config(). */
static void
-load_config(FILE *config_file, struct sset *remotes, struct sset *db_filenames)
+load_config(FILE *config_file, struct sset *remotes,
+ struct sset *db_filenames, bool *disable_epoll)
{
struct json *json;
@@ -1483,5 +1501,8 @@ load_config(FILE *config_file, struct sset *remotes,
struct sset *db_filenames)
sset_from_json(remotes, shash_find_data(json_object(json), "remotes"));
sset_from_json(db_filenames,
shash_find_data(json_object(json), "db_filenames"));
+ bool b = json_boolean(shash_find_data(json_object(json), "disable_epoll"));
+ *disable_epoll = b;
+
json_destroy(json);
}
diff --git a/tests/ovsdb-server.at b/tests/ovsdb-server.at
index c869d6f..9244382 100644
--- a/tests/ovsdb-server.at
+++ b/tests/ovsdb-server.at
@@ -277,6 +277,17 @@ ordinals
])
AT_CLEANUP
+AT_SETUP([ovsdb-server with --disable-epoll])
+AT_KEYWORDS([ovsdb server positive])
+AT_SKIP_IF([test "$IS_WIN32" = "yes"])
+# Start ovsdb-server, initially with one db, not using epoll poll group
+ordinal_schema > schema
+AT_CHECK([ovsdb-tool create db1 schema], [0], [ignore], [ignore])
+on_exit 'kill `cat *.pid`'
+AT_CHECK([ovsdb-server --detach --no-chdir --pidfile --disable-epoll
--remote=punix:socket db1], [0])
+AT_CHECK([ovsdb-client list-dbs unix:socket] , [0], [ignore], [ignore])
+AT_CLEANUP
+
AT_SETUP([ovsdb-server/add-db and remove-db with --monitor])
AT_KEYWORDS([ovsdb server positive])
AT_SKIP_IF([test "$IS_WIN32" = "yes"])
--
1.9.1
_______________________________________________
dev mailing list
dev@openvswitch.org
http://openvswitch.org/mailman/listinfo/dev
