The branch, master has been updated
via 49791db7dc74cffd7e88bd73091590cdc1909328 (commit)
via 4340263b219d75c39f8de22abe3f6f1c1ee63ea2 (commit)
from 02b62482164a3c69715949074feb7f191a29d534 (commit)
http://gitweb.samba.org/?p=ctdb.git;a=shortlog;h=master
- Log -----------------------------------------------------------------
commit 49791db7dc74cffd7e88bd73091590cdc1909328
Author: Ronnie Sahlberg <ronniesahlb...@gmail.com>
Date: Tue Feb 28 06:58:59 2012 +1100
Add a tunable variable to control how long we defer after a ctdb addip
until we force a rebalance and try to failback addresses onto this node
Have it default to 300 seconds.
commit 4340263b219d75c39f8de22abe3f6f1c1ee63ea2
Author: Ronnie Sahlberg <ronniesahlb...@gmail.com>
Date: Tue Feb 28 06:56:04 2012 +1100
When adding ips to nodes, set up a deferred rebalance for the whole node to
trigger after 60 seconds in case the normal ipreallocated is not sufficient to
trigger rebalance.
-----------------------------------------------------------------------
Summary of changes:
include/ctdb_private.h | 2 +
include/ctdb_protocol.h | 5 ++++
server/ctdb_recoverd.c | 56 +++++++++++++++++++++++++++++++++++++++++++++++
server/ctdb_takeover.c | 40 +++++++++++++++++++++++++++++++++
server/ctdb_tunables.c | 3 +-
tools/ctdb.c | 52 ++++++++++++++++++++++++++++++++++---------
6 files changed, 146 insertions(+), 12 deletions(-)
Changeset truncated at 500 lines:
diff --git a/include/ctdb_private.h b/include/ctdb_private.h
index 4b7e675..3039953 100644
--- a/include/ctdb_private.h
+++ b/include/ctdb_private.h
@@ -124,6 +124,7 @@ struct ctdb_tunable {
uint32_t lcp2_public_ip_assignment;
uint32_t allow_client_db_attach;
uint32_t recover_pdb_by_seqnum;
+ uint32_t deferred_rebalance_on_node_add;
};
/*
@@ -1132,6 +1133,7 @@ int ctdb_set_single_public_ip(struct ctdb_context *ctdb,
int ctdb_set_event_script(struct ctdb_context *ctdb, const char *script);
int ctdb_set_event_script_dir(struct ctdb_context *ctdb, const char
*script_dir);
int ctdb_set_notification_script(struct ctdb_context *ctdb, const char
*script);
+void lcp2_forcerebalance(struct ctdb_context *ctdb, uint32_t pnn);
int ctdb_takeover_run(struct ctdb_context *ctdb, struct ctdb_node_map
*nodemap);
int32_t ctdb_control_tcp_client(struct ctdb_context *ctdb, uint32_t client_id,
diff --git a/include/ctdb_protocol.h b/include/ctdb_protocol.h
index 4cf2789..3e466d8 100644
--- a/include/ctdb_protocol.h
+++ b/include/ctdb_protocol.h
@@ -130,6 +130,11 @@ struct ctdb_call_info {
*/
#define CTDB_SRVID_TAKEOVER_RUN 0xFB00000000000000LL
+/* request recovery daemon to rebalance ips for a node.
+ input is uint32_t for the node id.
+*/
+#define CTDB_SRVID_REBALANCE_NODE 0xFB01000000000000LL
+
/* A message id to ask the recovery daemon to temporarily disable the
public ip checks
*/
diff --git a/server/ctdb_recoverd.c b/server/ctdb_recoverd.c
index b5a6d63..642679b 100644
--- a/server/ctdb_recoverd.c
+++ b/server/ctdb_recoverd.c
@@ -65,6 +65,7 @@ struct ctdb_recoverd {
struct ip_reallocate_list *reallocate_callers;
TALLOC_CTX *ip_check_disable_ctx;
struct ctdb_control_get_ifaces *ifaces;
+ TALLOC_CTX *deferred_rebalance_ctx;
};
#define CONTROL_TIMEOUT() timeval_current_ofs(ctdb->tunable.recover_timeout, 0)
@@ -2068,6 +2069,57 @@ static void reenable_ip_check(struct event_context *ev,
struct timed_event *te,
}
+static void ctdb_rebalance_timeout(struct event_context *ev, struct
timed_event *te,
+ struct timeval t, void *p)
+{
+ struct ctdb_recoverd *rec = talloc_get_type(p, struct ctdb_recoverd);
+ struct ctdb_context *ctdb = rec->ctdb;
+ int ret;
+
+ DEBUG(DEBUG_NOTICE,("Rebalance all nodes that have had ip assignment
changes.\n"));
+
+ ret = ctdb_takeover_run(ctdb, rec->nodemap);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR, (__location__ " Unable to setup public
takeover addresses. ctdb_takeover_run() failed.\n"));
+ rec->need_takeover_run = true;
+ }
+
+ talloc_free(rec->deferred_rebalance_ctx);
+ rec->deferred_rebalance_ctx = NULL;
+}
+
+
+static void recd_node_rebalance_handler(struct ctdb_context *ctdb, uint64_t
srvid,
+ TDB_DATA data, void *private_data)
+{
+ uint32_t pnn;
+ struct ctdb_recoverd *rec = talloc_get_type(private_data, struct
ctdb_recoverd);
+
+ if (data.dsize != sizeof(uint32_t)) {
+ DEBUG(DEBUG_ERR,(__location__ " Incorrect size of node
rebalance message. Was %zd but expected %zd bytes\n", data.dsize,
sizeof(uint32_t)));
+ return;
+ }
+
+ if (ctdb->tunable.deferred_rebalance_on_node_add == 0) {
+ return;
+ }
+
+ pnn = *(uint32_t *)&data.dptr[0];
+
+ lcp2_forcerebalance(ctdb, pnn);
+ DEBUG(DEBUG_NOTICE,("Received message to perform node rebalancing for
node %d\n", pnn));
+
+ if (rec->deferred_rebalance_ctx != NULL) {
+ talloc_free(rec->deferred_rebalance_ctx);
+ }
+ rec->deferred_rebalance_ctx = talloc_new(rec);
+ event_add_timed(ctdb->ev, rec->deferred_rebalance_ctx,
+
timeval_current_ofs(ctdb->tunable.deferred_rebalance_on_node_add, 0),
+ ctdb_rebalance_timeout, rec);
+}
+
+
+
static void recd_update_ip_handler(struct ctdb_context *ctdb, uint64_t srvid,
TDB_DATA data, void *private_data)
{
@@ -3623,6 +3675,10 @@ static void monitor_cluster(struct ctdb_context *ctdb)
/* register a message port for updating the recovery daemons node
assignment for an ip */
ctdb_client_set_message_handler(ctdb, CTDB_SRVID_RECD_UPDATE_IP,
recd_update_ip_handler, rec);
+ /* register a message port for forcing a rebalance of a node next
+ reallocation */
+ ctdb_client_set_message_handler(ctdb, CTDB_SRVID_REBALANCE_NODE,
recd_node_rebalance_handler, rec);
+
for (;;) {
TALLOC_CTX *mem_ctx = talloc_new(ctdb);
struct timeval start;
diff --git a/server/ctdb_takeover.c b/server/ctdb_takeover.c
index c91d2f7..bff22a8 100644
--- a/server/ctdb_takeover.c
+++ b/server/ctdb_takeover.c
@@ -1531,6 +1531,32 @@ static bool basic_failback(struct ctdb_context *ctdb,
return false;
}
+struct ctdb_rebalancenodes {
+ struct ctdb_rebalancenodes *next;
+ uint32_t pnn;
+};
+static struct ctdb_rebalancenodes *force_rebalance_list = NULL;
+
+
+/* set this flag to force the node to be rebalanced even if it just didnt
+ become healthy again.
+*/
+void lcp2_forcerebalance(struct ctdb_context *ctdb, uint32_t pnn)
+{
+ struct ctdb_rebalancenodes *rebalance;
+
+ for (rebalance = force_rebalance_list; rebalance; rebalance =
rebalance->next) {
+ if (rebalance->pnn == pnn) {
+ return;
+ }
+ }
+
+ rebalance = talloc(ctdb, struct ctdb_rebalancenodes);
+ rebalance->pnn = pnn;
+ rebalance->next = force_rebalance_list;
+ force_rebalance_list = rebalance;
+}
+
/* Do necessary LCP2 initialisation. Bury it in a function here so
* that we can unit test it.
*/
@@ -1562,6 +1588,20 @@ static void lcp2_init(struct ctdb_context * tmp_ctx,
(*newly_healthy)[tmp_ip->pnn] = false;
}
}
+
+ /* 3rd step: if a node is forced to re-balance then
+ we allow failback onto the node */
+ while (force_rebalance_list != NULL) {
+ struct ctdb_rebalancenodes *next = force_rebalance_list->next;
+
+ if (force_rebalance_list->pnn <= nodemap->num) {
+ (*newly_healthy)[force_rebalance_list->pnn] = true;
+ }
+
+ DEBUG(DEBUG_ERR,("During ipreallocation, forced rebalance of
node %d\n", force_rebalance_list->pnn));
+ talloc_free(force_rebalance_list);
+ force_rebalance_list = next;
+ }
}
/* Allocate any unassigned addresses using the LCP2 algorithm to find
diff --git a/server/ctdb_tunables.c b/server/ctdb_tunables.c
index d7323ec..c5e15c1 100644
--- a/server/ctdb_tunables.c
+++ b/server/ctdb_tunables.c
@@ -72,7 +72,8 @@ static const struct {
{ "StatHistoryInterval", 1, offsetof(struct ctdb_tunable,
stat_history_interval), false },
{ "DeferredAttachTO", 120, offsetof(struct ctdb_tunable,
deferred_attach_timeout), false },
{ "AllowClientDBAttach", 1, offsetof(struct ctdb_tunable,
allow_client_db_attach), false },
- { "RecoverPDBBySeqNum", 0, offsetof(struct ctdb_tunable,
recover_pdb_by_seqnum), false }
+ { "RecoverPDBBySeqNum", 0, offsetof(struct ctdb_tunable,
recover_pdb_by_seqnum), false },
+ { "DeferredRebalanceOnNodeAdd", 300, offsetof(struct ctdb_tunable,
deferred_rebalance_on_node_add) }
};
/*
diff --git a/tools/ctdb.c b/tools/ctdb.c
index a9fbddc..ad30989 100644
--- a/tools/ctdb.c
+++ b/tools/ctdb.c
@@ -1570,6 +1570,44 @@ static int control_moveip(struct ctdb_context *ctdb, int
argc, const char **argv
return 0;
}
+static int rebalance_node(struct ctdb_context *ctdb, uint32_t pnn)
+{
+ uint32_t recmaster;
+ TDB_DATA data;
+
+ if (ctdb_ctrl_getrecmaster(ctdb, ctdb, TIMELIMIT(), pnn, &recmaster) !=
0) {
+ DEBUG(DEBUG_ERR, ("Unable to get recmaster from node %u\n",
pnn));
+ return -1;
+ }
+
+ data.dptr = (uint8_t *)&pnn;
+ data.dsize = sizeof(uint32_t);
+ if (ctdb_client_send_message(ctdb, recmaster,
CTDB_SRVID_REBALANCE_NODE, data) != 0) {
+ DEBUG(DEBUG_ERR,("Failed to send message to force node
reallocation\n"));
+ return -1;
+ }
+
+ return 0;
+}
+
+
+/*
+ rebalance a node by setting it to allow failback and triggering a
+ takeover run
+ */
+static int control_rebalancenode(struct ctdb_context *ctdb, int argc, const
char **argv)
+{
+ switch (options.pnn) {
+ case CTDB_BROADCAST_ALL:
+ case CTDB_CURRENT_NODE:
+ DEBUG(DEBUG_ERR,("You must specify a node number with -n <pnn>
for the node to rebalance\n"));
+ return -1;
+ }
+
+ return rebalance_node(ctdb, options.pnn);
+}
+
+
static int getips_store_callback(void *param, void *data)
{
struct ctdb_public_ip *node_ip = (struct ctdb_public_ip *)data;
@@ -1937,17 +1975,8 @@ static int control_addip(struct ctdb_context *ctdb, int
argc, const char **argv)
return ret;
}
- do {
- ret = control_ipreallocate(ctdb, argc, argv);
- if (ret != 0) {
- DEBUG(DEBUG_ERR, ("IP Reallocate failed on node %u.
Wait 3 seconds and try again.\n", options.pnn));
- sleep(3);
- retries++;
- }
- } while (retries < 5 && ret != 0);
- if (ret != 0) {
- DEBUG(DEBUG_ERR, ("IP Reallocate failed on node %u. Giving
up.\n", options.pnn));
- talloc_free(tmp_ctx);
+ if (rebalance_node(ctdb, options.pnn) != 0) {
+ DEBUG(DEBUG_ERR,("Error when trying to rebalance node\n"));
return ret;
}
@@ -5512,6 +5541,7 @@ static const struct {
{ "readkey", control_readkey, true, false, "read
the content off a database key", "<tdb-file> <key>" },
{ "writekey", control_writekey, true, false, "write
to a database key", "<tdb-file> <key> <value>" },
{ "checktcpport", control_chktcpport, false, true, "check
if a service is bound to a specific tcp port or not", "<port>" },
+ { "rebalancenode", control_rebalancenode, false, false, "release
a node by allowing it to takeover ips", "<pnn>"},
{ "getdbseqnum", control_getdbseqnum, false, false, "get the
sequence number off a database", "<dbid>" },
{ "nodestatus", control_nodestatus, true, false, "show
and return node status" },
{ "dbstatistics", control_dbstatistics, false, false, "show db
statistics", "<db>" },
--
CTDB repository
