The branch, 1.2.40 has been updated via 0e99ca1cdf28a7043554afb78bd439f727ab4f95 (commit) via e80b2c15bf8c8fb5c3793acfebbe09d3cdd617b7 (commit) from 8b2d84482bacd3b31db013496ce82c2e7b730e86 (commit)
http://gitweb.samba.org/?p=ctdb.git;a=shortlog;h=1.2.40 - Log ----------------------------------------------------------------- commit 0e99ca1cdf28a7043554afb78bd439f727ab4f95 Author: Amitay Isaacs <ami...@gmail.com> Date: Wed Dec 5 11:38:42 2012 +1100 scripts: Add helper script to log locking information using /proc/locks This finds any processes locking tdb databases used by CTDB and logs stack trace for each process. Signed-off-by: Amitay Isaacs <ami...@gmail.com> commit e80b2c15bf8c8fb5c3793acfebbe09d3cdd617b7 Author: Amitay Isaacs <ami...@gmail.com> Date: Wed Dec 5 11:37:26 2012 +1100 daemon: Run an external script if freeze locks were not obtained during recovery If the freeze child is already created in ctdb_start_freeze(), then it indicates that the child process has not yet obtained the locks. This may be because another process has locked the databases and has not yet released the locks. In this case, invoke a helper script defined by environmental variable CTDB_DEBUG_LOCKS, to log information about locks. Signed-off-by: Amitay Isaacs <ami...@gmail.com> ----------------------------------------------------------------------- Summary of changes: Makefile.in | 1 + config/debug_locks.sh | 38 ++++++++++++++++++++++++++++++++++++++ packaging/RPM/ctdb.spec.in | 1 + server/ctdb_freeze.c | 24 ++++++++++++++++++++++++ 4 files changed, 64 insertions(+), 0 deletions(-) create mode 100755 config/debug_locks.sh Changeset truncated at 500 lines: diff --git a/Makefile.in b/Makefile.in index e3d6d5b..136ae30 100755 --- a/Makefile.in +++ b/Makefile.in @@ -264,6 +264,7 @@ install: all ${INSTALLCMD} -m 644 config/functions $(DESTDIR)$(etcdir)/ctdb ${INSTALLCMD} -m 755 config/statd-callout $(DESTDIR)$(etcdir)/ctdb ${INSTALLCMD} -m 755 config/interface_modify.sh $(DESTDIR)$(etcdir)/ctdb + ${INSTALLCMD} -m 755 config/debug_locks.sh $(DESTDIR)$(etcdir)/ctdb ${INSTALLCMD} -m 644 config/events.d/README $(DESTDIR)$(docdir)/ctdb/README.eventscripts ${INSTALLCMD} -m 644 doc/recovery-process.txt $(DESTDIR)$(docdir)/ctdb/recovery-process.txt ${INSTALLCMD} -m 755 config/events.d/00.ctdb $(DESTDIR)$(etcdir)/ctdb/events.d diff --git a/config/debug_locks.sh b/config/debug_locks.sh new file mode 100755 index 0000000..5b9efed --- /dev/null +++ b/config/debug_locks.sh @@ -0,0 +1,38 @@ +#!/bin/sh + +# Create sed expression to convert inodes to names +sed_cmd=$( ls -li /var/ctdb/*.tdb.* /var/ctdb/persistent/*.tdb.* | + sed -e "s#/var/ctdb[/persistent]*/\(.*\)#\1#" | + awk '{printf "s#[0-9]*:[0-9]*:%s #%s #\n", $1, $10}' ) + +# Parse /proc/locks and extract following information +# pid process_name tdb_name offsets [W] +out=$( cat /proc/locks | + grep -F "POSIX ADVISORY WRITE" | + awk '{ if($2 == "->") { print $6, $7, $8, $9, "W" } else { print $5, $6, $7, $8 } }' | + while read pid rest ; do + pname=$(readlink /proc/$pid/exe) + echo $pid $pname $rest + done | sed -e "$sed_fu" | grep "\.tdb" ) + +if [ -n "$out" ]; then + # Log information about locks + echo "$out" | logger -t "debug-lock" + + # Find processes that are waiting for locks + dbs=$(echo "$out" | grep "W$" | awk '{print $3}') + all_pids="" + for db in $dbs ; do + pids=$(echo "$out" | grep -v "W$" | grep "$db" | grep -v ctdbd | awk '{print $1}') + all_pids="$all_pids $pids" + done + pids=$(echo $all_pids | sort -u) + + # For each process waiting, log stack trace + for pid in $pids ; do + gstack $pid | logger -t "debug-lock $pid" +# gcore -o /var/log/core-deadlock-ctdb $pid + done +fi + +exit 0 diff --git a/packaging/RPM/ctdb.spec.in b/packaging/RPM/ctdb.spec.in index de8193d..39a5df9 100644 --- a/packaging/RPM/ctdb.spec.in +++ b/packaging/RPM/ctdb.spec.in @@ -113,6 +113,7 @@ rm -rf $RPM_BUILD_ROOT %{_sysconfdir}/ctdb/events.d/91.lvs %{_sysconfdir}/ctdb/statd-callout %{_sysconfdir}/ctdb/interface_modify.sh +%{_sysconfdir}/ctdb/debug_locks.sh %{_sbindir}/ctdbd %{_bindir}/ctdb %{_bindir}/smnotify diff --git a/server/ctdb_freeze.c b/server/ctdb_freeze.c index 0f70fd3..f422e6d 100644 --- a/server/ctdb_freeze.c +++ b/server/ctdb_freeze.c @@ -256,6 +256,26 @@ static int ctdb_freeze_waiter_destructor(struct ctdb_freeze_waiter *w) } /* + * Run an external script to check if there is a deadlock situation + */ +static void ctdb_debug_locks(void) +{ + const char *cmd = getenv("CTDB_DEBUG_LOCKS"); + int pid; + + if (cmd == NULL) { + return; + } + + pid = fork(); + + /* Execute only in child process */ + if (pid == 0) { + execl(cmd, cmd, NULL); + } +} + +/* start the freeze process for a certain priority */ int ctdb_start_freeze(struct ctdb_context *ctdb, uint32_t priority) @@ -283,6 +303,10 @@ int ctdb_start_freeze(struct ctdb_context *ctdb, uint32_t priority) ctdb->freeze_handles[priority] = ctdb_freeze_lock(ctdb, priority); CTDB_NO_MEMORY(ctdb, ctdb->freeze_handles[priority]); ctdb->freeze_mode[priority] = CTDB_FREEZE_PENDING; + } else { + /* The previous free lock child has not yet been able to get locks. + * Invoke debugging script */ + ctdb_debug_locks(); } return 0; -- CTDB repository