The branch, master has been updated
via 54e24a151d2163954e5a2a1c0f41a2b5c19ae44b (commit)
via 718233c445cd6627ab3962b6565c2655f1f8efd0 (commit)
via a5c6bb1fffb8dc3960af113957a1fd080cc7c245 (commit)
from f243a916ee71013f7402b9c396c2ead88eb3aab0 (commit)
http://gitweb.samba.org/?p=ctdb.git;a=shortlog;h=master
- Log -----------------------------------------------------------------
commit 54e24a151d2163954e5a2a1c0f41a2b5c19ae44b
Author: Amitay Isaacs <ami...@gmail.com>
Date: Wed Nov 14 15:51:59 2012 +1100
locking: Do not use RECLOCK for tracking DB locks and latencies
RECLOCK is for recovery lock in CTDB. Do not override the meaning for
tracking locks on databases. Database lock latency has nothing to do
with recovery lock latency.
Signed-off-by: Amitay Isaacs <ami...@gmail.com>
commit 718233c445cd6627ab3962b6565c2655f1f8efd0
Author: Amitay Isaacs <ami...@gmail.com>
Date: Tue Nov 6 17:06:54 2012 +1100
tools/ctdb: Do not use function return value as pnn
This fixes the wrong code where same variable 'ret' is used to track the pnn
and the return value of a function call.
Signed-off-by: Amitay Isaacs <ami...@gmail.com>
commit a5c6bb1fffb8dc3960af113957a1fd080cc7c245
Author: Amitay Isaacs <ami...@gmail.com>
Date: Tue Oct 23 16:23:12 2012 +1100
recoverd: Track the nodes that fail takeover run and set culprit count
If any of the nodes fail takeover run (either due to timeout or failure
to complete within takeover_timeout interval) from main loop, recovery
master will give up trying takeover run with following message:
"Unable to setup public takeover addresses. Try again later"
And as a side-effect the monitoring is disabled on all the nodes. Before
ctdb_takeover_run() is called from main loop, monitoring get disabled via
startrecovery event. Since ctdb_takeover_run() fails, it never runs
recovered event and monitoring does not get re-enabled.
In main_loop, ctdb_takeover_run() is called with a takeover_fail_callback.
This callback will get called if any of the nodes fail in handling
takeip/releaseip/ipreallocated events in ctdb_takeover_run().
Signed-off-by: Amitay Isaacs <ami...@gmail.com>
-----------------------------------------------------------------------
Summary of changes:
include/ctdb_private.h | 14 +++++++-------
server/ctdb_lock.c | 10 ++++++++--
server/ctdb_recoverd.c | 34 +++++++++++++++++++++++++++++-----
server/ctdb_takeover.c | 14 +++++++++++---
tools/ctdb.c | 8 +++++---
5 files changed, 60 insertions(+), 20 deletions(-)
Changeset truncated at 500 lines:
diff --git a/include/ctdb_private.h b/include/ctdb_private.h
index 2da9c50..f06a1f1 100644
--- a/include/ctdb_private.h
+++ b/include/ctdb_private.h
@@ -365,7 +365,7 @@ struct ctdb_daemon_data {
}
\
}
-#define CTDB_UPDATE_DB_RECLOCK_LATENCY(ctdb_db, name, counter, value) \
+#define CTDB_UPDATE_DB_LATENCY(ctdb_db, operation, counter, value)
\
{
\
if (value > ctdb_db->statistics.counter.max)
\
ctdb_db->statistics.counter.max = value;
\
@@ -377,10 +377,10 @@ struct ctdb_daemon_data {
ctdb_db->statistics.counter.num++;
\
\
if (ctdb_db->ctdb->tunable.reclock_latency_ms != 0) {
\
- if (value*1000 >
ctdb_db->ctdb->tunable.reclock_latency_ms) { \
+ if (value*1000 > ctdb_db->ctdb->tunable.log_latency_ms)
{ \
DEBUG(DEBUG_ERR,
\
- ("High RECLOCK latency %fs for operation
%s\n", \
- value, name));
\
+ ("High latency %.6fs for operation %s on
database %s\n",\
+ value, operation, ctdb_db->db_name));
\
}
\
}
\
}
@@ -1195,6 +1195,8 @@ char *ctdb_get_process_name(pid_t pid);
bool ctdb_get_lock_info(pid_t req_pid, struct ctdb_lock_info *lock_info);
bool ctdb_get_blocker_pid(struct ctdb_lock_info *reqlock, pid_t *blocker_pid);
+typedef void (*client_async_callback)(struct ctdb_context *ctdb, uint32_t
node_pnn, int32_t res, TDB_DATA outdata, void *callback_data);
+
int ctdb_set_public_addresses(struct ctdb_context *ctdb, bool check_addresses);
int ctdb_set_single_public_ip(struct ctdb_context *ctdb,
const char *iface,
@@ -1204,7 +1206,7 @@ int ctdb_set_event_script_dir(struct ctdb_context *ctdb,
const char *script_dir)
int ctdb_set_debug_hung_script(struct ctdb_context *ctdb, const char *script);
int ctdb_set_notification_script(struct ctdb_context *ctdb, const char
*script);
void lcp2_forcerebalance(struct ctdb_context *ctdb, uint32_t pnn);
-int ctdb_takeover_run(struct ctdb_context *ctdb, struct ctdb_node_map
*nodemap);
+int ctdb_takeover_run(struct ctdb_context *ctdb, struct ctdb_node_map
*nodemap, client_async_callback fail_callback, void *callback_data);
int32_t ctdb_control_tcp_client(struct ctdb_context *ctdb, uint32_t client_id,
TDB_DATA indata);
@@ -1343,8 +1345,6 @@ int32_t ctdb_monitoring_mode(struct ctdb_context *ctdb);
int ctdb_set_child_logging(struct ctdb_context *ctdb);
void ctdb_lockdown_memory(struct ctdb_context *ctdb);
-typedef void (*client_async_callback)(struct ctdb_context *ctdb, uint32_t
node_pnn, int32_t res, TDB_DATA outdata, void *callback_data);
-
struct client_async_data {
enum ctdb_controls opcode;
bool dont_log_errors;
diff --git a/server/ctdb_lock.c b/server/ctdb_lock.c
index 81f0eb3..e78f3fc 100644
--- a/server/ctdb_lock.c
+++ b/server/ctdb_lock.c
@@ -53,6 +53,13 @@ enum lock_type {
LOCK_ALLDB,
};
+static const char * const lock_type_str[] = {
+ "lock_record",
+ "lock_db",
+ "lock_alldb_prio",
+ "lock_db",
+};
+
struct lock_request;
/* lock_context is the common part for a lock request */
@@ -567,11 +574,10 @@ static void ctdb_lock_handler(struct tevent_context *ev,
if (locked) {
CTDB_INCREMENT_STAT(lock_ctx->ctdb, locks.num_current);
- CTDB_UPDATE_RECLOCK_LATENCY(lock_ctx->ctdb, "lock()",
locks.latency, t);
CTDB_INCREMENT_STAT(lock_ctx->ctdb, locks.buckets[id]);
if (lock_ctx->ctdb_db) {
CTDB_INCREMENT_DB_STAT(lock_ctx->ctdb_db,
locks.num_current);
- CTDB_UPDATE_DB_RECLOCK_LATENCY(lock_ctx->ctdb_db,
"lock()", locks.latency, t);
+ CTDB_UPDATE_DB_LATENCY(lock_ctx->ctdb_db,
lock_type_str[lock_ctx->type], locks.latency, t);
CTDB_INCREMENT_DB_STAT(lock_ctx->ctdb_db,
locks.buckets[id]);
}
} else {
diff --git a/server/ctdb_recoverd.c b/server/ctdb_recoverd.c
index 699aace..6d0dbc4 100644
--- a/server/ctdb_recoverd.c
+++ b/server/ctdb_recoverd.c
@@ -1493,6 +1493,21 @@ static int sync_recovery_lock_file_across_cluster(struct
ctdb_recoverd *rec)
/*
+ * this callback is called for every node that failed to execute
ctdb_takeover_run()
+ * and set flag to re-run takeover run.
+ */
+static void takeover_fail_callback(struct ctdb_context *ctdb, uint32_t
node_pnn, int32_t res, TDB_DATA outdata, void *callback_data)
+{
+ struct ctdb_recoverd *rec = talloc_get_type(callback_data, struct
ctdb_recoverd);
+
+ DEBUG(DEBUG_ERR, (__location__ " Node %u failed the takeover run.
Setting it as recovery fail culprit\n", node_pnn));
+
+ ctdb_set_culprit(rec, node_pnn);
+ rec->need_takeover_run = true;
+}
+
+
+/*
we are the recmaster, and recovery is needed - start a recovery run
*/
static int do_recovery(struct ctdb_recoverd *rec,
@@ -1781,7 +1796,7 @@ static int do_recovery(struct ctdb_recoverd *rec,
return -1;
}
rec->need_takeover_run = false;
- ret = ctdb_takeover_run(ctdb, nodemap);
+ ret = ctdb_takeover_run(ctdb, nodemap, NULL, NULL);
if (ret != 0) {
DEBUG(DEBUG_ERR, (__location__ " Unable to setup public
takeover addresses. ctdb_takeover_run() failed.\n"));
rec->need_takeover_run = true;
@@ -2140,7 +2155,7 @@ static void ctdb_rebalance_timeout(struct event_context
*ev, struct timed_event
DEBUG(DEBUG_NOTICE,("Rebalance all nodes that have had ip assignment
changes.\n"));
- ret = ctdb_takeover_run(ctdb, rec->nodemap);
+ ret = ctdb_takeover_run(ctdb, rec->nodemap, NULL, NULL);
if (ret != 0) {
DEBUG(DEBUG_ERR, (__location__ " Unable to setup public
takeover addresses. ctdb_takeover_run() failed.\n"));
rec->need_takeover_run = true;
@@ -2366,7 +2381,7 @@ static void process_ipreallocate_requests(struct
ctdb_context *ctdb, struct ctdb
rec->need_takeover_run = true;
}
if (ret == 0) {
- ret = ctdb_takeover_run(ctdb, rec->nodemap);
+ ret = ctdb_takeover_run(ctdb, rec->nodemap, NULL, NULL);
if (ret != 0) {
DEBUG(DEBUG_ERR,("Failed to reallocate addresses:
ctdb_takeover_run() failed.\n"));
rec->need_takeover_run = true;
@@ -3789,9 +3804,18 @@ static void main_loop(struct ctdb_context *ctdb, struct
ctdb_recoverd *rec,
return;
}
- ret = ctdb_takeover_run(ctdb, nodemap);
+ /* If takeover run fails, then the offending nodes are
+ * assigned ban culprit counts. And we re-try takeover.
+ * If takeover run fails repeatedly, the node would get
+ * banned.
+ *
+ * If rec->need_takeover_run is not set to true at this
+ * failure, monitoring is disabled cluster-wide (via
+ * startrecovery eventscript) and will not get enabled.
+ */
+ ret = ctdb_takeover_run(ctdb, nodemap, takeover_fail_callback,
rec);
if (ret != 0) {
- DEBUG(DEBUG_ERR, (__location__ " Unable to setup public
takeover addresses. Try again later\n"));
+ DEBUG(DEBUG_ERR, (__location__ " Unable to setup public
takeover addresses. Trying again\n"));
return;
}
diff --git a/server/ctdb_takeover.c b/server/ctdb_takeover.c
index 4fcddd1..d324e3f 100644
--- a/server/ctdb_takeover.c
+++ b/server/ctdb_takeover.c
@@ -2148,7 +2148,8 @@ static void noiptakeover_cb(struct ctdb_context *ctdb,
uint32_t pnn, int32_t res
/*
make any IP alias changes for public addresses that are necessary
*/
-int ctdb_takeover_run(struct ctdb_context *ctdb, struct ctdb_node_map *nodemap)
+int ctdb_takeover_run(struct ctdb_context *ctdb, struct ctdb_node_map *nodemap,
+ client_async_callback fail_callback, void *callback_data)
{
int i;
struct ctdb_public_ip ip;
@@ -2217,6 +2218,9 @@ int ctdb_takeover_run(struct ctdb_context *ctdb, struct
ctdb_node_map *nodemap)
async_data = talloc_zero(tmp_ctx, struct client_async_data);
CTDB_NO_MEMORY_FATAL(ctdb, async_data);
+ async_data->fail_callback = fail_callback;
+ async_data->callback_data = callback_data;
+
for (i=0;i<nodemap->num;i++) {
/* don't talk to unconnected nodes, but do talk to banned nodes
*/
if (nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED) {
@@ -2274,6 +2278,10 @@ int ctdb_takeover_run(struct ctdb_context *ctdb, struct
ctdb_node_map *nodemap)
/* tell all nodes to get their own IPs */
async_data = talloc_zero(tmp_ctx, struct client_async_data);
CTDB_NO_MEMORY_FATAL(ctdb, async_data);
+
+ async_data->fail_callback = fail_callback;
+ async_data->callback_data = callback_data;
+
for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
if (tmp_ip->pnn == -1) {
/* this IP won't be taken over */
@@ -2331,8 +2339,8 @@ ipreallocated:
if (ctdb_client_async_control(ctdb, CTDB_CONTROL_RUN_EVENTSCRIPTS,
nodes, 0, TAKEOVER_TIMEOUT(),
false, data,
- NULL, NULL,
- NULL) != 0) {
+ NULL, fail_callback,
+ callback_data) != 0) {
DEBUG(DEBUG_ERR, (__location__ " failed to send control to run
eventscripts with \"ipreallocated\"\n"));
}
diff --git a/tools/ctdb.c b/tools/ctdb.c
index d132cae..a974fb2 100644
--- a/tools/ctdb.c
+++ b/tools/ctdb.c
@@ -2255,10 +2255,12 @@ static int control_delip(struct ctdb_context *ctdb, int
argc, const char **argv)
}
if (ips->ips[i].pnn == options.pnn) {
- ret = find_other_host_for_public_ip(ctdb, &addr);
- if (ret != -1) {
+ int pnn;
+
+ pnn = find_other_host_for_public_ip(ctdb, &addr);
+ if (pnn != -1) {
do {
- ret = move_ip(ctdb, &addr, ret);
+ ret = move_ip(ctdb, &addr, pnn);
if (ret != 0) {
DEBUG(DEBUG_ERR,("Failed to move ip to
node %d. Wait 3 seconds and try again.\n", options.pnn));
sleep(3);
--
CTDB repository
