The branch, master has been updated
via 11734be353a1e246163eda631d35dfe55d1d6fb1 (commit)
via 3e09f25d419635f6dd679b48fa65370f7860be7d (commit)
from 5402f85dde045576cbaf64e01c68e28ed52204e8 (commit)
http://gitweb.samba.org/?p=ctdb.git;a=shortlog;h=master
- Log -----------------------------------------------------------------
commit 11734be353a1e246163eda631d35dfe55d1d6fb1
Author: Amitay Isaacs <ami...@gmail.com>
Date: Fri Feb 22 12:59:39 2013 +1100
common/io: For scheduling immediate events use tevent_schedule_immediate
tevent_schedule_immediate() is much more efficient at handling events that
need
to be processed immediately rather than creating timed events with
timeval_zero().
Signed-off-by: Amitay Isaacs <ami...@gmail.com>
commit 3e09f25d419635f6dd679b48fa65370f7860be7d
Author: Amitay Isaacs <ami...@gmail.com>
Date: Thu Feb 21 13:16:15 2013 +1100
ctdbd: Add an index db for message list for faster searches
When CTDB is busy with lots of smbd, CTDB was spending too much time in
daemon_check_srvids() which searches a list of srvids in the registered
message handlers. Using a hash based index significantly improves the
performance of search in a linked list.
Signed-off-by: Amitay Isaacs <ami...@gmail.com>
-----------------------------------------------------------------------
Summary of changes:
common/ctdb_io.c | 26 ++++---
common/ctdb_message.c | 205 +++++++++++++++++++++++++++++++++++++++++++++---
include/ctdb_private.h | 13 +++-
server/ctdb_daemon.c | 8 +--
4 files changed, 218 insertions(+), 34 deletions(-)
Changeset truncated at 500 lines:
diff --git a/common/ctdb_io.c b/common/ctdb_io.c
index b4224c4..4e592b9 100644
--- a/common/ctdb_io.c
+++ b/common/ctdb_io.c
@@ -45,6 +45,7 @@ struct ctdb_queue_pkt {
struct ctdb_queue {
struct ctdb_context *ctdb;
+ struct tevent_immediate *im;
struct ctdb_buffer buffer; /* input buffer */
struct ctdb_queue_pkt *out_queue, *out_queue_tail;
uint32_t out_queue_length;
@@ -66,8 +67,8 @@ int ctdb_queue_length(struct ctdb_queue *queue)
static void queue_process(struct ctdb_queue *queue);
-static void queue_process_event(struct event_context *ev, struct timed_event
*te,
- struct timeval t, void *private_data)
+static void queue_process_event(struct tevent_context *ev, struct
tevent_immediate *im,
+ void *private_data)
{
struct ctdb_queue *queue = talloc_get_type(private_data, struct
ctdb_queue);
@@ -118,9 +119,9 @@ static void queue_process(struct ctdb_queue *queue)
queue->buffer.length -= pkt_size;
if (queue->buffer.length > 0) {
- /* There is more data to be processed, setup timed event */
- event_add_timed(queue->ctdb->ev, queue, timeval_zero(),
- queue_process_event, queue);
+ /* There is more data to be processed, schedule an event */
+ tevent_schedule_immediate(queue->im, queue->ctdb->ev,
+ queue_process_event, queue);
}
/* It is the responsibility of the callback to free 'data' */
@@ -192,8 +193,8 @@ failed:
/* used when an event triggers a dead queue */
-static void queue_dead(struct event_context *ev, struct timed_event *te,
- struct timeval t, void *private_data)
+static void queue_dead(struct event_context *ev, struct tevent_immediate *im,
+ void *private_data)
{
struct ctdb_queue *queue = talloc_get_type(private_data, struct
ctdb_queue);
queue->callback(NULL, 0, queue->private_data);
@@ -224,8 +225,8 @@ static void queue_io_write(struct ctdb_queue *queue)
talloc_free(queue->fde);
queue->fde = NULL;
queue->fd = -1;
- event_add_timed(queue->ctdb->ev, queue, timeval_zero(),
- queue_dead, queue);
+ tevent_schedule_immediate(queue->im, queue->ctdb->ev,
+ queue_dead, queue);
return;
}
if (n <= 0) return;
@@ -291,8 +292,8 @@ int ctdb_queue_send(struct ctdb_queue *queue, uint8_t
*data, uint32_t length)
talloc_free(queue->fde);
queue->fde = NULL;
queue->fd = -1;
- event_add_timed(queue->ctdb->ev, queue, timeval_zero(),
- queue_dead, queue);
+ tevent_schedule_immediate(queue->im, queue->ctdb->ev,
+ queue_dead, queue);
/* yes, we report success, as the dead node is
handled via a separate event */
return 0;
@@ -402,6 +403,9 @@ struct ctdb_queue *ctdb_queue_setup(struct ctdb_context
*ctdb,
va_end(ap);
CTDB_NO_MEMORY_NULL(ctdb, queue->name);
+ queue->im= tevent_create_immediate(queue);
+ CTDB_NO_MEMORY_NULL(ctdb, queue->im);
+
queue->ctdb = ctdb;
queue->fd = fd;
queue->alignment = alignment;
diff --git a/common/ctdb_message.c b/common/ctdb_message.c
index 5fc7b22..50d2189 100644
--- a/common/ctdb_message.c
+++ b/common/ctdb_message.c
@@ -2,6 +2,7 @@
ctdb_message protocol code
Copyright (C) Andrew Tridgell 2007
+ Copyright (C) Amitay Isaacs 2013
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -27,16 +28,103 @@
#include "../include/ctdb_private.h"
#include "lib/util/dlinklist.h"
+static int message_list_db_init(struct ctdb_context *ctdb)
+{
+ ctdb->message_list_indexdb = tdb_open("messagedb", 8192,
+ TDB_INTERNAL|TDB_DISALLOW_NESTING,
+ O_RDWR|O_CREAT, 0);
+ if (ctdb->message_list_indexdb == NULL) {
+ DEBUG(DEBUG_ERR, ("Failed to create message list indexdb\n"));
+ return -1;
+ }
+
+ return 0;
+}
+
+static int message_list_db_add(struct ctdb_context *ctdb, TDB_DATA key,
TDB_DATA data)
+{
+ int ret;
+
+ if (ctdb->message_list_indexdb == NULL) {
+ ret = message_list_db_init(ctdb);
+ if (ret < 0) {
+ return -1;
+ }
+ }
+
+ ret = tdb_store(ctdb->message_list_indexdb, key, data, TDB_INSERT);
+ if (ret < 0) {
+ DEBUG(DEBUG_ERR, ("Failed to add message list handler (%s)\n",
+ tdb_errorstr(ctdb->message_list_indexdb)));
+ return -1;
+ }
+
+ return 0;
+}
+
+static int message_list_db_delete(struct ctdb_context *ctdb, TDB_DATA key)
+{
+ int ret;
+
+ if (ctdb->message_list_indexdb == NULL) {
+ return -1;
+ }
+
+ ret = tdb_delete(ctdb->message_list_indexdb, key);
+ if (ret < 0) {
+ DEBUG(DEBUG_ERR, ("Failed to delete message list handler
(%s)\n",
+ tdb_errorstr(ctdb->message_list_indexdb)));
+ return -1;
+ }
+
+ return 0;
+}
+
+static int message_list_db_fetch(struct ctdb_context *ctdb, TDB_DATA key,
TDB_DATA *data)
+{
+ if (ctdb->message_list_indexdb == NULL) {
+ return -1;
+ }
+
+ *data = tdb_fetch(ctdb->message_list_indexdb, key);
+ if (data->dsize == 0) {
+ return -1;
+ }
+ return 0;
+}
+
/*
this dispatches the messages to the registered ctdb message handler
*/
int ctdb_dispatch_message(struct ctdb_context *ctdb, uint64_t srvid, TDB_DATA
data)
{
- struct ctdb_message_list *ml;
+ struct ctdb_message_list_header *h;
+ struct ctdb_message_list *m;
+ TDB_DATA key, hdata;
+ uint64_t srvid_all = CTDB_SRVID_ALL;
+ int ret;
+
+ key.dptr = (uint8_t *)&srvid;
+ key.dsize = sizeof(uint64_t);
+
+ ret = message_list_db_fetch(ctdb, key, &hdata);
+ if (ret == 0) {
+ h = *(struct ctdb_message_list_header **)hdata.dptr;
- for (ml=ctdb->message_list;ml;ml=ml->next) {
- if (ml->srvid == srvid || ml->srvid == CTDB_SRVID_ALL) {
- ml->message_handler(ctdb, srvid, data,
ml->message_private);
+ for (m=h->m; m; m=m->next) {
+ m->message_handler(ctdb, srvid, data,
m->message_private);
+ }
+ }
+
+ key.dptr = (uint8_t *)&srvid_all;
+ key.dsize = sizeof(uint64_t);
+
+ ret = message_list_db_fetch(ctdb, key, &hdata);
+ if (ret == 0) {
+ h = *(struct ctdb_message_list_header **)hdata.dptr;
+
+ for(m=h->m; m; m=m->next) {
+ m->message_handler(ctdb, srvid, data,
m->message_private);
}
}
@@ -57,13 +145,37 @@ void ctdb_request_message(struct ctdb_context *ctdb,
struct ctdb_req_header *hdr
ctdb_dispatch_message(ctdb, c->srvid, data);
}
+/*
+ * When header is freed, remove all the srvid handlers
+ */
+static int message_header_destructor(struct ctdb_message_list_header *h)
+{
+ struct ctdb_message_list *m;
+ TDB_DATA key;
+
+ while (h->m != NULL) {
+ m = h->m;
+ DLIST_REMOVE(h->m, m);
+ TALLOC_FREE(m);
+ }
+
+ key.dptr = (uint8_t *)&h->srvid;
+ key.dsize = sizeof(uint64_t);
+
+ message_list_db_delete(h->ctdb, key);
+ DLIST_REMOVE(h->ctdb->message_list_header, h);
+
+ return 0;
+}
/*
when a client goes away, we need to remove its srvid handler from the list
*/
static int message_handler_destructor(struct ctdb_message_list *m)
{
- DLIST_REMOVE(m->ctdb->message_list, m);
+ struct ctdb_message_list_header *h = m->h;
+
+ DLIST_REMOVE(h->m, m);
return 0;
}
@@ -76,20 +188,47 @@ int ctdb_register_message_handler(struct ctdb_context
*ctdb,
ctdb_msg_fn_t handler,
void *private_data)
{
+ struct ctdb_message_list_header *h;
struct ctdb_message_list *m;
+ TDB_DATA key, data;
+ int ret;
- m = talloc(mem_ctx, struct ctdb_message_list);
+ m = talloc_zero(mem_ctx, struct ctdb_message_list);
CTDB_NO_MEMORY(ctdb, m);
- m->ctdb = ctdb;
- m->srvid = srvid;
m->message_handler = handler;
m->message_private = private_data;
-
- DLIST_ADD(ctdb->message_list, m);
- talloc_set_destructor(m, message_handler_destructor);
+ key.dptr = (uint8_t *)&srvid;
+ key.dsize = sizeof(uint64_t);
+
+ ret = message_list_db_fetch(ctdb, key, &data);
+ if (ret < 0) {
+ /* srvid not registered yet */
+ h = talloc_zero(ctdb, struct ctdb_message_list_header);
+ CTDB_NO_MEMORY(ctdb, h);
+
+ h->ctdb = ctdb;
+ h->srvid = srvid;
+
+ data.dptr = (uint8_t *)&h;
+ data.dsize = sizeof(struct ctdb_message_list_header *);
+ ret = message_list_db_add(ctdb, key, data);
+ if (ret < 0) {
+ talloc_free(m);
+ talloc_free(h);
+ return -1;
+ }
+ DLIST_ADD(ctdb->message_list_header, h);
+ talloc_set_destructor(h, message_header_destructor);
+ } else {
+ h = *(struct ctdb_message_list_header **)data.dptr;
+ }
+
+ m->h = h;
+ DLIST_ADD(h->m, m);
+ talloc_set_destructor(m, message_handler_destructor);
return 0;
}
@@ -99,13 +238,53 @@ int ctdb_register_message_handler(struct ctdb_context
*ctdb,
*/
int ctdb_deregister_message_handler(struct ctdb_context *ctdb, uint64_t srvid,
void *private_data)
{
+ struct ctdb_message_list_header *h;
struct ctdb_message_list *m;
+ TDB_DATA key, data;
+ int ret;
+
+ key.dptr = (uint8_t *)&srvid;
+ key.dsize = sizeof(uint64_t);
+
+ ret = message_list_db_fetch(ctdb, key, &data);
+ if (ret < 0) {
+ return -1;
+ }
- for (m=ctdb->message_list;m;m=m->next) {
- if (m->srvid == srvid && m->message_private == private_data) {
+ h = *(struct ctdb_message_list_header **)data.dptr;
+ for (m=h->m; m; m=m->next) {
+ if (m->message_private == private_data) {
talloc_free(m);
+ if (h->m == NULL) {
+ talloc_free(h);
+ }
return 0;
}
}
+
return -1;
}
+
+
+/*
+ * check if the given srvid exists
+ */
+bool ctdb_check_message_handler(struct ctdb_context *ctdb, uint64_t srvid)
+{
+ struct ctdb_message_list_header *h;
+ TDB_DATA key, data;
+
+ key.dptr = (uint8_t *)&srvid;
+ key.dsize = sizeof(uint64_t);
+
+ if (message_list_db_fetch(ctdb, key, &data) < 0) {
+ return false;
+ }
+
+ h = *(struct ctdb_message_list_header **)data.dptr;
+ if (h->m == NULL) {
+ return false;
+ }
+
+ return true;
+}
diff --git a/include/ctdb_private.h b/include/ctdb_private.h
index 7794c4c..7e46603 100644
--- a/include/ctdb_private.h
+++ b/include/ctdb_private.h
@@ -286,10 +286,15 @@ struct ctdb_upcalls {
/* list of message handlers - needs to be changed to a more efficient data
structure so we can find a message handler given a srvid quickly */
-struct ctdb_message_list {
+struct ctdb_message_list_header {
+ struct ctdb_message_list_header *next, *prev;
struct ctdb_context *ctdb;
- struct ctdb_message_list *next, *prev;
uint64_t srvid;
+ struct ctdb_message_list *m;
+};
+struct ctdb_message_list {
+ struct ctdb_message_list *next, *prev;
+ struct ctdb_message_list_header *h;
ctdb_msg_fn_t message_handler;
void *message_private;
};
@@ -478,7 +483,8 @@ struct ctdb_context {
const struct ctdb_upcalls *upcalls; /* transport upcalls */
void *private_data; /* private to transport */
struct ctdb_db_context *db_list;
- struct ctdb_message_list *message_list;
+ struct ctdb_message_list_header *message_list_header;
+ struct tdb_context *message_list_indexdb;
struct ctdb_daemon_data daemon;
struct ctdb_statistics statistics;
struct ctdb_statistics statistics_current;
@@ -1014,6 +1020,7 @@ int32_t ctdb_control_traverse_kill(struct ctdb_context
*ctdb, TDB_DATA indata,
TDB_DATA *outdata, uint32_t srcnode);
int ctdb_dispatch_message(struct ctdb_context *ctdb, uint64_t srvid, TDB_DATA
data);
+bool ctdb_check_message_handler(struct ctdb_context *ctdb, uint64_t srvid);
int daemon_register_message_handler(struct ctdb_context *ctdb, uint32_t
client_id, uint64_t srvid);
int ctdb_deregister_message_handler(struct ctdb_context *ctdb, uint64_t srvid,
void *private_data);
diff --git a/server/ctdb_daemon.c b/server/ctdb_daemon.c
index 0550fab..983f62d 100644
--- a/server/ctdb_daemon.c
+++ b/server/ctdb_daemon.c
@@ -225,13 +225,7 @@ int daemon_check_srvids(struct ctdb_context *ctdb,
TDB_DATA indata,
return -1;
}
for (i=0; i<num_ids; i++) {
- struct ctdb_message_list *ml;
- for (ml=ctdb->message_list; ml; ml=ml->next) {
- if (ml->srvid == ids[i]) {
- break;
- }
- }
- if (ml != NULL) {
+ if (ctdb_check_message_handler(ctdb, ids[i])) {
results[i/8] |= (1 << (i%8));
}
}
--
CTDB repository
