The branch, master has been updated
via 0566ef3d6cef809bda204877c493c80ff9eb2c40 (commit)
via b05ccf366df985e0a3365aacc75761ebd438deaf (commit)
via eeaabd579841f60ab2c5b004cbbb1f5de2bfe685 (commit)
via 05c934b10ad2690be9d75c9033a0b849bf16455d (commit)
from 81663b81687c0ba681500cca6aa8174bb9587ad2 (commit)
http://gitweb.samba.org/?p=sahlberg/ctdb.git;a=shortlog;h=master
- Log -----------------------------------------------------------------
commit 0566ef3d6cef809bda204877c493c80ff9eb2c40
Author: Ronnie Sahlberg <ronniesahlb...@gmail.com>
Date: Tue Mar 1 12:09:42 2011 +1100
If/when the recovery daemon terminates unexpectedly, try to restart it
again from the main daemon instead of just shutting down the main deamon too.
While it does not address the reason for recovery daemon shutting down, it
reduces the impact of such issues and makes the system more robust.
commit b05ccf366df985e0a3365aacc75761ebd438deaf
Author: Ronnie Sahlberg <ronniesahlb...@gmail.com>
Date: Fri Feb 25 10:33:12 2011 +1100
ATTACH_DB: simplify the code slightly and change the semantics to only
refuse a db attach during recovery IF we can associate the request from a
genuine real client instead of deciding this on whether client_id is zero or
This will suppress/avoid messages like these :
DB Attach to database %s refused. Can not match clientid...
commit eeaabd579841f60ab2c5b004cbbb1f5de2bfe685
Author: Ronnie Sahlberg <ronniesahlb...@gmail.com>
Date: Wed Feb 23 15:46:36 2011 +1100
Deferred attach : at early startup, defer any db attach calls until we are
out of recovery.
commit 05c934b10ad2690be9d75c9033a0b849bf16455d
Author: Ronnie Sahlberg <ronniesahlb...@gmail.com>
Date: Fri Feb 25 10:06:08 2011 +1100
Dont return error if trying to set db priority on a db that does not yet
exist.
Just treat as a nop.
When the database is created later it will get its priority set properly.
-----------------------------------------------------------------------
Summary of changes:
include/ctdb_private.h | 11 ++++-
server/ctdb_control.c | 4 +-
server/ctdb_ltdb_server.c | 107 +++++++++++++++++++++++++++++++++++++++-----
server/ctdb_recover.c | 10 ++++
server/ctdb_recoverd.c | 25 ++++++----
server/ctdb_tunables.c | 3 +-
6 files changed, 133 insertions(+), 27 deletions(-)
Changeset truncated at 500 lines:
diff --git a/include/ctdb_private.h b/include/ctdb_private.h
index db5594d..68877ec 100644
--- a/include/ctdb_private.h
+++ b/include/ctdb_private.h
@@ -118,6 +118,7 @@ struct ctdb_tunable {
uint32_t use_status_events_for_monitoring;
uint32_t allow_unhealthy_db_read;
uint32_t stat_history_interval;
+ uint32_t deferred_attach_timeout;
};
/*
@@ -488,6 +489,9 @@ struct ctdb_context {
/* used in the recovery daemon to remember the ip allocation */
struct trbt_tree *ip_tree;
+
+ /* Used to defer db attach requests while in recovery mode */
+ struct ctdb_deferred_attach_context *deferred_attach;
};
struct ctdb_db_context {
@@ -800,7 +804,10 @@ int ctdb_daemon_send_control(struct ctdb_context *ctdb,
uint32_t destnode,
void *private_data);
int32_t ctdb_control_db_attach(struct ctdb_context *ctdb, TDB_DATA indata,
- TDB_DATA *outdata, uint64_t tdb_flags, bool
persistent);
+ TDB_DATA *outdata, uint64_t tdb_flags,
+ bool persistent, uint32_t client_id,
+ struct ctdb_req_control *c,
+ bool *async_reply);
int ctdb_daemon_set_call(struct ctdb_context *ctdb, uint32_t db_id,
ctdb_fn_t fn, int id);
@@ -1366,4 +1373,6 @@ int32_t ctdb_control_get_stat_history(struct ctdb_context
*ctdb,
int ctdb_deferred_drop_all_ips(struct ctdb_context *ctdb);
+int ctdb_process_deferred_attach(struct ctdb_context *ctdb);
+
#endif
diff --git a/server/ctdb_control.c b/server/ctdb_control.c
index 90900c9..69724e3 100644
--- a/server/ctdb_control.c
+++ b/server/ctdb_control.c
@@ -221,10 +221,10 @@ static int32_t ctdb_control_dispatch(struct ctdb_context
*ctdb,
}
case CTDB_CONTROL_DB_ATTACH:
- return ctdb_control_db_attach(ctdb, indata, outdata, srvid,
false);
+ return ctdb_control_db_attach(ctdb, indata, outdata, srvid, false,
client_id, c, async_reply);
case CTDB_CONTROL_DB_ATTACH_PERSISTENT:
- return ctdb_control_db_attach(ctdb, indata, outdata, srvid,
true);
+ return ctdb_control_db_attach(ctdb, indata, outdata, srvid, true,
client_id, c, async_reply);
case CTDB_CONTROL_SET_CALL: {
struct ctdb_control_set_call *sc =
diff --git a/server/ctdb_ltdb_server.c b/server/ctdb_ltdb_server.c
index ba2a9cb..19a68ec 100644
--- a/server/ctdb_ltdb_server.c
+++ b/server/ctdb_ltdb_server.c
@@ -745,16 +745,107 @@ again:
}
+struct ctdb_deferred_attach_context {
+ struct ctdb_deferred_attach_context *next, *prev;
+ struct ctdb_context *ctdb;
+ struct ctdb_req_control *c;
+};
+
+
+static int ctdb_deferred_attach_destructor(struct ctdb_deferred_attach_context
*da_ctx)
+{
+ DLIST_REMOVE(da_ctx->ctdb->deferred_attach, da_ctx);
+
+ return 0;
+}
+
+static void ctdb_deferred_attach_timeout(struct event_context *ev, struct
timed_event *te, struct timeval t, void *private_data)
+{
+ struct ctdb_deferred_attach_context *da_ctx =
talloc_get_type(private_data, struct ctdb_deferred_attach_context);
+ struct ctdb_context *ctdb = da_ctx->ctdb;
+
+ ctdb_request_control_reply(ctdb, da_ctx->c, NULL, -1, NULL);
+ talloc_free(da_ctx);
+}
+
+static void ctdb_deferred_attach_callback(struct event_context *ev, struct
timed_event *te, struct timeval t, void *private_data)
+{
+ struct ctdb_deferred_attach_context *da_ctx =
talloc_get_type(private_data, struct ctdb_deferred_attach_context);
+ struct ctdb_context *ctdb = da_ctx->ctdb;
+
+ /* This talloc-steals the packet ->c */
+ ctdb_input_pkt(ctdb, (struct ctdb_req_header *)da_ctx->c);
+ talloc_free(da_ctx);
+}
+
+int ctdb_process_deferred_attach(struct ctdb_context *ctdb)
+{
+ struct ctdb_deferred_attach_context *da_ctx;
+
+ /* call it from the main event loop as soon as the current event
+ finishes.
+ */
+ while ((da_ctx = ctdb->deferred_attach) != NULL) {
+ DLIST_REMOVE(ctdb->deferred_attach, da_ctx);
+ event_add_timed(ctdb->ev, ctdb, timeval_current_ofs(1,0),
ctdb_deferred_attach_callback, da_ctx);
+ }
+
+ return 0;
+}
+
/*
a client has asked to attach a new database
*/
int32_t ctdb_control_db_attach(struct ctdb_context *ctdb, TDB_DATA indata,
TDB_DATA *outdata, uint64_t tdb_flags,
- bool persistent)
+ bool persistent, uint32_t client_id,
+ struct ctdb_req_control *c,
+ bool *async_reply)
{
const char *db_name = (const char *)indata.dptr;
struct ctdb_db_context *db;
struct ctdb_node *node = ctdb->nodes[ctdb->pnn];
+ struct ctdb_client *client = NULL;
+
+ /* dont allow any local clients to attach while we are in recovery mode
+ * except for the recovery daemon.
+ * allow all attach from the network since these are always from remote
+ * recovery daemons.
+ */
+ if (client_id != 0) {
+ client = ctdb_reqid_find(ctdb, client_id, struct ctdb_client);
+ }
+ if (client != NULL) {
+ /* If the node is inactive it is not part of the cluster
+ and we should not allow clients to attach to any
+ databases
+ */
+ if (node->flags & NODE_FLAGS_INACTIVE) {
+ DEBUG(DEBUG_ERR,("DB Attach to database %s refused
since node is inactive (disconnected or banned)\n", db_name));
+ return -1;
+ }
+
+ if (ctdb->recovery_mode == CTDB_RECOVERY_ACTIVE
+ && client->pid != ctdb->recoverd_pid) {
+ struct ctdb_deferred_attach_context *da_ctx =
talloc(client, struct ctdb_deferred_attach_context);
+
+ if (da_ctx == NULL) {
+ DEBUG(DEBUG_ERR,("DB Attach to database %s
deferral for client with pid:%d failed due to OOM.\n", db_name, client->pid));
+ return -1;
+ }
+
+ da_ctx->ctdb = ctdb;
+ da_ctx->c = talloc_steal(da_ctx, c);
+ talloc_set_destructor(da_ctx,
ctdb_deferred_attach_destructor);
+ DLIST_ADD(ctdb->deferred_attach, da_ctx);
+
+ event_add_timed(ctdb->ev, da_ctx,
timeval_current_ofs(ctdb->tunable.deferred_attach_timeout, 0),
ctdb_deferred_attach_timeout, da_ctx);
+
+ DEBUG(DEBUG_ERR,("DB Attach to database %s deferred for
client with pid:%d since node is in recovery mode.\n", db_name, client->pid));
+ *async_reply = true;
+ return 0;
+ }
+ }
/* the client can optionally pass additional tdb flags, but we
only allow a subset of those on the database in ctdb. Note
@@ -762,16 +853,6 @@ int32_t ctdb_control_db_attach(struct ctdb_context *ctdb,
TDB_DATA indata,
srvid to the attach control */
tdb_flags &= (TDB_NOSYNC|TDB_INCOMPATIBLE_HASH);
- /* If the node is inactive it is not part of the cluster
- and we should not allow clients to attach to any
- databases
- */
- if (node->flags & NODE_FLAGS_INACTIVE) {
- DEBUG(DEBUG_ERR,("DB Attach to database %s refused since node
is inactive (disconnected or banned)\n", db_name));
- return -1;
- }
-
-
/* see if we already have this name */
db = ctdb_db_handle(ctdb, db_name);
if (db) {
@@ -1111,12 +1192,12 @@ int32_t ctdb_control_set_db_priority(struct
ctdb_context *ctdb, TDB_DATA indata)
ctdb_db = find_ctdb_db(ctdb, db_prio->db_id);
if (!ctdb_db) {
DEBUG(DEBUG_ERR,("Unknown db_id 0x%x in
ctdb_set_db_priority\n", db_prio->db_id));
- return -1;
+ return 0;
}
if ((db_prio->priority<1) || (db_prio->priority>NUM_DB_PRIORITIES)) {
DEBUG(DEBUG_ERR,("Trying to set invalid priority : %u\n",
db_prio->priority));
- return -1;
+ return 0;
}
ctdb_db->priority = db_prio->priority;
diff --git a/server/ctdb_recover.c b/server/ctdb_recover.c
index 2dbfbd4..0cbd7dc 100644
--- a/server/ctdb_recover.c
+++ b/server/ctdb_recover.c
@@ -630,6 +630,11 @@ static void set_recmode_handler(struct event_context *ev,
struct fd_event *fde,
state->ctdb->recovery_mode = state->recmode;
+ /* release any deferred attach calls from clients */
+ if (state->recmode == CTDB_RECOVERY_NORMAL) {
+ ctdb_process_deferred_attach(state->ctdb);
+ }
+
ctdb_request_control_reply(state->ctdb, state->c, NULL, 0, NULL);
talloc_free(state);
return;
@@ -716,6 +721,11 @@ int32_t ctdb_control_set_recmode(struct ctdb_context *ctdb,
state->fd[0] = -1;
state->fd[1] = -1;
+ /* release any deferred attach calls from clients */
+ if (recmode == CTDB_RECOVERY_NORMAL) {
+ ctdb_process_deferred_attach(ctdb);
+ }
+
if (ctdb->tunable.verify_recovery_lock == 0) {
/* dont need to verify the reclock file */
ctdb->recovery_mode = recmode;
diff --git a/server/ctdb_recoverd.c b/server/ctdb_recoverd.c
index d75370d..cc0be36 100644
--- a/server/ctdb_recoverd.c
+++ b/server/ctdb_recoverd.c
@@ -70,6 +70,7 @@ struct ctdb_recoverd {
#define CONTROL_TIMEOUT() timeval_current_ofs(ctdb->tunable.recover_timeout, 0)
#define MONITOR_TIMEOUT() timeval_current_ofs(ctdb->tunable.recover_interval,
0)
+static void ctdb_restart_recd(struct event_context *ev, struct timed_event
*te, struct timeval t, void *private_data);
/*
ban a node for a period of time
@@ -3521,18 +3522,12 @@ static void ctdb_check_recd(struct event_context *ev,
struct timed_event *te,
struct ctdb_context *ctdb = talloc_get_type(p, struct ctdb_context);
if (kill(ctdb->recoverd_pid, 0) != 0) {
- DEBUG(DEBUG_ERR,("Recovery daemon (pid:%d) is no longer
running. Shutting down main daemon\n", (int)ctdb->recoverd_pid));
+ DEBUG(DEBUG_ERR,("Recovery daemon (pid:%d) is no longer
running. Trying to restart recovery daemon.\n", (int)ctdb->recoverd_pid));
- ctdb_stop_recoverd(ctdb);
- ctdb_stop_keepalive(ctdb);
- ctdb_stop_monitoring(ctdb);
- ctdb_release_all_ips(ctdb);
- if (ctdb->methods != NULL) {
- ctdb->methods->shutdown(ctdb);
- }
- ctdb_event_script(ctdb, CTDB_EVENT_SHUTDOWN);
+ event_add_timed(ctdb->ev, ctdb, timeval_zero(),
+ ctdb_restart_recd, ctdb);
- exit(10);
+ return;
}
event_add_timed(ctdb->ev, ctdb,
@@ -3634,3 +3629,13 @@ void ctdb_stop_recoverd(struct ctdb_context *ctdb)
DEBUG(DEBUG_NOTICE,("Shutting down recovery daemon\n"));
kill(ctdb->recoverd_pid, SIGTERM);
}
+
+static void ctdb_restart_recd(struct event_context *ev, struct timed_event
*te,
+ struct timeval t, void *private_data)
+{
+ struct ctdb_context *ctdb = talloc_get_type(private_data, struct
ctdb_context);
+
+ DEBUG(DEBUG_ERR,("Restarting recovery daemon\n"));
+ ctdb_stop_recoverd(ctdb);
+ ctdb_start_recoverd(ctdb);
+}
diff --git a/server/ctdb_tunables.c b/server/ctdb_tunables.c
index 4cd1b45..0f8d7c8 100644
--- a/server/ctdb_tunables.c
+++ b/server/ctdb_tunables.c
@@ -65,7 +65,8 @@ static const struct {
{ "MaxQueueDropMsg", 1000000, offsetof(struct ctdb_tunable,
max_queue_depth_drop_msg) },
{ "UseStatusEvents", 0, offsetof(struct ctdb_tunable,
use_status_events_for_monitoring) },
{ "AllowUnhealthyDBRead", 0, offsetof(struct ctdb_tunable,
allow_unhealthy_db_read) },
- { "StatHistoryInterval", 1, offsetof(struct ctdb_tunable,
stat_history_interval) }
+ { "StatHistoryInterval", 1, offsetof(struct ctdb_tunable,
stat_history_interval) },
+ { "DeferredAttachTO", 120, offsetof(struct ctdb_tunable,
deferred_attach_timeout) }
};
/*
--
CTDB repository
