The branch, master has been updated via 533ad24 ctdb-tests: Do not mix bool and int data types via 30f7d7d ctdb-tests: Use ctdb_fetch_lock instead of ctdb_fetch_lock_readonly via 7801532 ctdb-client: ctdb_fetch_lock should check for readonly delegations via aa7cd51 ctdb-tests: Fix and extend read-only records test via 234f8eb ctdb-tests: Add a new NFS tickle test from f99a759 ctdb-locking: Instead of comparing key, compare key hash
http://gitweb.samba.org/?p=samba.git;a=shortlog;h=master - Log ----------------------------------------------------------------- commit 533ad246443bbea5116c06fb6478453ddf004080 Author: Amitay Isaacs <ami...@gmail.com> Date: Thu Mar 27 15:07:58 2014 +1100 ctdb-tests: Do not mix bool and int data types Signed-off-by: Amitay Isaacs <ami...@gmail.com> Reviewed-by: Martin Schwenke <mar...@meltin.net> Autobuild-User(master): Amitay Isaacs <ami...@samba.org> Autobuild-Date(master): Fri Mar 28 07:56:18 CET 2014 on sn-devel-104 commit 30f7d7db8c0bf370464238133f75b5e0bdce37c6 Author: Amitay Isaacs <ami...@gmail.com> Date: Thu Mar 27 15:11:09 2014 +1100 ctdb-tests: Use ctdb_fetch_lock instead of ctdb_fetch_lock_readonly This will test that ctdb_fetch_lock correctly revokes readonly delegations. Signed-off-by: Amitay Isaacs <ami...@gmail.com> Reviewed-by: Martin Schwenke <mar...@meltin.net> commit 78015320b60b0fd0d8c3dc65fbbe3e38e4a02993 Author: Amitay Isaacs <ami...@gmail.com> Date: Thu Mar 27 15:06:58 2014 +1100 ctdb-client: ctdb_fetch_lock should check for readonly delegations When readonly delegations were added, ctdb_fetch_lock code should have been modified to include the check for readonly flags. Signed-off-by: Amitay Isaacs <ami...@gmail.com> Reviewed-by: Martin Schwenke <mar...@meltin.net> commit aa7cd51bbc438555552584a7d71f5bae6909603f Author: Martin Schwenke <mar...@meltin.net> Date: Thu Mar 27 16:26:21 2014 +1100 ctdb-tests: Fix and extend read-only records test This test currently counts the number of read-only-enabled databases and expects there to only be 1. It fails when there are existing databases with read-only already enabled. Instead, check just the test database. Clean up the test by adding some functions to check for precisely the read-only flags that should be set on a node after each operation. Signed-off-by: Martin Schwenke <mar...@meltin.net> Reviewed-by: Amitay Isaacs <ami...@gmail.com> commit 234f8eb5712c38872444c5dd7a258903b389b062 Author: Martin Schwenke <mar...@meltin.net> Date: Fri Feb 28 15:54:54 2014 +1100 ctdb-tests: Add a new NFS tickle test This one ensures that a newly started node gets an up-to-date tickle list. Tweak some of the integration test functions to accommodate this. Signed-off-by: Martin Schwenke <mar...@meltin.net> Reviewed-by: Amitay Isaacs <ami...@gmail.com> ----------------------------------------------------------------------- Summary of changes: ctdb/client/ctdb_client.c | 15 ++ ctdb/tests/complex/34_nfs_tickle_restart.sh | 98 +++++++++++ ctdb/tests/scripts/integration.bash | 8 +- ctdb/tests/simple/75_readonly_records_basic.sh | 216 ++++++++++++++---------- ctdb/tests/src/ctdb_fetch_readonly_loop.c | 2 +- ctdb/tests/src/ctdb_update_record.c | 2 +- 6 files changed, 242 insertions(+), 99 deletions(-) create mode 100755 ctdb/tests/complex/34_nfs_tickle_restart.sh Changeset truncated at 500 lines: diff --git a/ctdb/client/ctdb_client.c b/ctdb/client/ctdb_client.c index 885dbfd..7560115 100644 --- a/ctdb/client/ctdb_client.c +++ b/ctdb/client/ctdb_client.c @@ -709,6 +709,21 @@ again: goto again; } + /* if this is a request for read/write and we have delegations + we have to revoke all delegations first + */ + if ((h->header.dmaster == ctdb_db->ctdb->pnn) && + (h->header.flags & CTDB_REC_RO_HAVE_DELEGATIONS)) { + ctdb_ltdb_unlock(ctdb_db, key); + ret = ctdb_client_force_migration(ctdb_db, key); + if (ret != 0) { + DEBUG(DEBUG_DEBUG,("ctdb_fetch_readonly_lock: force_migration failed\n")); + talloc_free(h); + return NULL; + } + goto again; + } + DEBUG(DEBUG_DEBUG,("ctdb_fetch_lock: we are dmaster - done\n")); return h; } diff --git a/ctdb/tests/complex/34_nfs_tickle_restart.sh b/ctdb/tests/complex/34_nfs_tickle_restart.sh new file mode 100755 index 0000000..93587e2 --- /dev/null +++ b/ctdb/tests/complex/34_nfs_tickle_restart.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify that a newly started CTDB node gets updated tickle details + +Prerequisites: + +* An active CTDB cluster with at least 2 nodes with public addresses. + +* Test must be run on a real or virtual cluster rather than against + local daemons. + +* Cluster nodes must be listening on the NFS TCP port (2049). + +Steps: + +As with 31_nfs_tickle.sh but restart a node after the tickle is +registered. + +Expected results: + +* CTDB should correctly communicated tickles to new CTDB instances as + they join the cluster. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +set -e + +ctdb_test_init "$@" + +ctdb_test_check_real_cluster + +cluster_is_healthy + +# Reset configuration +ctdb_restart_when_done + +# We need this for later, so we know how long to run nc for. +try_command_on_node any $CTDB getvar MonitorInterval +monitor_interval="${out#*= }" +#echo "Monitor interval on node $test_node is $monitor_interval seconds." + +select_test_node_and_ips +try_command_on_node $test_node "$CTDB listnodes -Y" +listnodes_output="$out" +numnodes=$(wc -l <<<"$listnodes_output") + +test_port=2049 + +echo "Connecting to node ${test_node} on IP ${test_ip}:${test_port} with netcat..." + +nc -d -w 600 $test_ip $test_port & +nc_pid=$! +ctdb_test_exit_hook_add "kill $nc_pid >/dev/null 2>&1" + +wait_until_get_src_socket "tcp" "${test_ip}:${test_port}" $nc_pid "nc" +src_socket="$out" +echo "Source socket is $src_socket" + +wait_for_monitor_event $test_node + +echo "Wait until NFS connection is tracked by CTDB on test node ..." +wait_until 10 check_tickles $test_node $test_ip $test_port $src_socket + +echo "Select a node to restart ctdbd" +rn=$(awk -F: -v test_node=$test_node \ + '$2 != test_node { print $2 ; exit }' <<<"$listnodes_output") + +echo "Restarting CTDB on node ${rn}" +try_command_on_node $rn $CTDB_TEST_WRAPPER restart_ctdb_1 + +# In some theoretical world this is racy. In practice, the node will +# take quite a while to become healthy, so this will beat any +# assignment of IPs to the node. +echo "Setting NoIPTakeover on node ${rn}" +try_command_on_node $rn $CTDB setvar NoIPTakeover 1 + +wait_until_healthy + +echo "Getting TickleUpdateInterval..." +try_command_on_node $test_node $CTDB getvar TickleUpdateInterval +update_interval="$out" + +echo "Wait until NFS connection is tracked by CTDB on all nodes..." +if ! wait_until $(($update_interval * 2)) \ + check_tickles_all $numnodes $test_ip $test_port $src_socket ; then + echo "BAD: connection not tracked on all nodes:" + echo "$out" + exit 1 +fi + +# We could go on to test whether the tickle ACK gets sent. However, +# this is tested in previous tests and the use of NoIPTakeover +# complicates things on a 2 node cluster. diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index 665fc7d..1ff02d5 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -278,7 +278,7 @@ wait_until_healthy () echo "Waiting for cluster to become healthy..." - wait_until 120 _cluster_is_healthy + wait_until $timeout onnode -q any $CTDB_TEST_WRAPPER _cluster_is_healthy } # This function is becoming nicely overloaded. Soon it will collapse! :-) @@ -438,7 +438,7 @@ _ctdb_hack_options () esac } -_restart_ctdb () +restart_ctdb_1 () { _ctdb_hack_options "$@" @@ -452,7 +452,7 @@ _restart_ctdb () # Restart CTDB on all nodes. Override for local daemons. _restart_ctdb_all () { - onnode -p all $CTDB_TEST_WRAPPER _restart_ctdb "$@" + onnode -p all $CTDB_TEST_WRAPPER restart_ctdb_1 "$@" } # Nothing needed for a cluster. Override for local daemons. @@ -479,7 +479,7 @@ restart_ctdb () continue } - onnode -q 1 $CTDB_TEST_WRAPPER wait_until_healthy || { + wait_until_healthy || { echo "Cluster didn't become healthy. Restarting..." continue } diff --git a/ctdb/tests/simple/75_readonly_records_basic.sh b/ctdb/tests/simple/75_readonly_records_basic.sh index 80a6adc..6cd2cce 100755 --- a/ctdb/tests/simple/75_readonly_records_basic.sh +++ b/ctdb/tests/simple/75_readonly_records_basic.sh @@ -3,11 +3,11 @@ test_info() { cat <<EOF -Readonly records can be activated at runtime using a ctdb command. -If readonly records are not activated, then any attempt to fetch a readonly +Read-only records can be activated at runtime using a ctdb command. +If read-only records are not activated, then any attempt to fetch a read-only copy should be automatically upgraded to a read-write fetch_lock(). -If readonly delegations are present, then any attempt to aquire a read-write +If read-only delegations are present, then any attempt to aquire a read-write fetch_lock will trigger all delegations to be revoked before the fetch lock completes. @@ -20,21 +20,16 @@ Steps: 1. Verify that the status on all of the ctdb nodes is 'OK'. 2. create a test database and some records -3. try to fetch readonly records, this should not result in any delegations -4. activate readonly support -5. try to fetch readonly records, this should result in delegations +3. try to fetch read-only records, this should not result in any delegations +4. activate read-only support +5. try to fetch read-only records, this should result in delegations 6. do a fetchlock and the delegations should be revoked -7. try to fetch readonly records, this should result in delegations +7. try to fetch read-only records, this should result in delegations 8. do a recovery and the delegations should be revoked Expected results: -3. No delegations created when db is not in readonly mode -4. It is possible to activate readonly support for a database -5. Delegations should be created -6. Delegations should be revoked -8. Delegations should be revoked - +Delegations should be created and revoked as above EOF } @@ -50,114 +45,149 @@ cluster_is_healthy # Reset configuration ctdb_restart_when_done -try_command_on_node 0 "$CTDB listnodes" -num_nodes=$(echo "$out" | wc -l) - - -# create a temporary database to test with -echo create test database test.tdb -try_command_on_node 0 $CTDB attach test.tdb - - -# create some records -try_command_on_node all $CTDB_TEST_WRAPPER ctdb_update_record - -# -# 3 -# try readonly requests -echo Try some readonly fetches, these should all be upgraded to full fetchlocks -try_command_on_node 0,1,2 $CTDB_TEST_WRAPPER "ctdb_fetch_readonly_once </dev/null" +###################################################################### -# no delegations should have been created -numreadonly=`try_command_on_node -v all $CTDB cattdb test.tdb | grep READONLY | wc -l` -[ "$numreadonly" != "0" ] && { - echo "BAD: readonly delegations were created, but the feature is not activated on the database" - exit 1 +# Confirm that no nodes have databases with read-only delegations +check_no_readonly () +{ + try_command_on_node all $CTDB cattdb $testdb + local ro_flags="RO_HAVE_READONLY|RO_HAVE_DELEGATIONS" + local numreadonly=$(grep -c -E "$ro_flags" <<<"$out") || true + if [ $numreadonly -eq 0 ] ; then + echo "GOOD: no read-only delegations" + else + echo "BAD: there are read-only delegations" + echo "$out" + exit 1 + fi } +# Check that the test record has the correct read-only flags on the +# given nodes. The first node is the dmaster, which should know there +# are delegations but should not be flagged as having a read-only +# copy. Subsequent nodes should have a read-only copy but not know +# about any (other) delegations. +check_readonly () +{ + local dmaster="$1" ; shift + local others="$*" + + local count + + try_command_on_node $dmaster $CTDB cattdb $testdb + count=$(grep -c -E "RO_HAVE_DELEGATIONS" <<<"$out") || true + if [ $count -eq 1 ] ; then + echo "GOOD: dmaster ${dmaster} has read-only delegations" + else + echo "BAD: dmaster ${dmaster} has no read-only delegations" + echo "$out" + exit 1 + fi + count=$(grep -c -E "RO_HAVE_READONLY" <<<"$out") || true + if [ $count -ne 0 ] ; then + echo "BAD: dmaster ${dmaster} has a read-only copy" + echo "$out" + exit 1 + fi + + local o + for o in $others ; do + try_command_on_node $o $CTDB cattdb $testdb + count=$(grep -c -E "RO_HAVE_READONLY" <<<"$out") || true + if [ $count -eq 1 ] ; then + echo "GOOD: node ${o} has a read-only copy" + else + echo "BAD: node ${o} has no read-only copy" + echo "$out" + exit 1 + fi + count=$(grep -c -E "RO_HAVE_DELEGATIONS" <<<"$out") || true + if [ $count -ne 0 ] ; then + echo "BAD: other node ${o} has read-only delegations" + echo "$out" + exit 1 + fi + done +} -# -# 4 -# +###################################################################### -echo Activating ReadOnly record support for test.tdb ... -# activate readonly support -try_command_on_node all $CTDB setdbreadonly test.tdb -numreadonly=`try_command_on_node -v 0 $CTDB getdbmap | grep READONLY | wc -l` -[ "$numreadonly" != "1" ] && { - echo BAD: could not activate readonly support for the test database - exit 1 -} +echo "Get list of nodes..." +try_command_on_node any $CTDB -Y listnodes +all_nodes=$(awk -F: '{print $2}' <<<"$out") +###################################################################### +testdb="test.tdb" +echo "Create test database \"${testdb}\"" +try_command_on_node 0 $CTDB attach $testdb -# -# 5 -# +echo "Create some records..." +try_command_on_node all $CTDB_TEST_WRAPPER ctdb_update_record -echo Create some readonly delegations ... -# fetch record to node 0 and make it dmaster -try_command_on_node 1 $CTDB_TEST_WRAPPER ctdb_update_record +###################################################################### -# fetch readonly to node 1 -try_command_on_node -v 0 $CTDB_TEST_WRAPPER "ctdb_fetch_readonly_once </dev/null" +echo "Try some readonly fetches, these should all be upgraded to full fetchlocks..." +try_command_on_node all $CTDB_TEST_WRAPPER "ctdb_fetch_readonly_once </dev/null" -numreadonly=`try_command_on_node -v all $CTDB cattdb test.tdb | grep RO_HAVE | wc -l` -[ "$numreadonly" != "2" ] && { - echo BAD: could not create readonly delegation - exit 1 -} +check_no_readonly +###################################################################### +echo "Activate read-only record support for \"$testdb\"..." +try_command_on_node all $CTDB setdbreadonly $testdb +# Database should be tagged as READONLY +try_command_on_node 0 $CTDB getdbmap +db_details=$(awk -v db="$testdb" '$2 == foo="name:" db { print }' <<<"$out") +if grep -q "READONLY" <<<"$db_details" ; then + echo "GOOD: read-only record support is enabled" +else + echo "BAD: could not activate read-only support" + echo "$db_details" + exit 1 +fi -# -# 6 -# +###################################################################### -echo verify that a fetchlock will revoke the delegations ... -# fetch record to node 0 and make it dmaster +echo "Create 1 read-only delegation ..." +# dmaster=1 try_command_on_node 1 $CTDB_TEST_WRAPPER ctdb_update_record -numreadonly=`try_command_on_node -v all $CTDB cattdb test.tdb | grep RO_HAVE | wc -l` -[ "$numreadonly" != "0" ] && { - echo BAD: fetchlock did not revoke delegations - exit 1 -} +# Fetch read-only to node 0 +try_command_on_node 0 $CTDB_TEST_WRAPPER "ctdb_fetch_readonly_once </dev/null" +check_readonly 1 0 -# -# 7 -# +###################################################################### -echo Create some readonly delegations ... -# fetch record to node 0 and make it dmaster +echo "Verify that a fetchlock revokes read-only delegations..." +# Node 1 becomes dmaster try_command_on_node 1 $CTDB_TEST_WRAPPER ctdb_update_record -# fetch readonly to node 1 -try_command_on_node -v 0 $CTDB_TEST_WRAPPER "ctdb_fetch_readonly_once </dev/null" +check_no_readonly -numreadonly=`try_command_on_node -v all $CTDB cattdb test.tdb | grep RO_HAVE | wc -l` -[ "$numreadonly" != "2" ] && { - echo BAD: could not create readonly delegation - exit 1 -} +###################################################################### +echo "Create more read-only delegations..." +dmaster=1 +try_command_on_node $dmaster $CTDB_TEST_WRAPPER ctdb_update_record +others="" +for n in $all_nodes ; do + if [ "$n" != "$dmaster" ] ; then + # Fetch read-only copy to this node + try_command_on_node $n \ + $CTDB_TEST_WRAPPER "ctdb_fetch_readonly_once </dev/null" + others="${others} ${n}" + fi +done +check_readonly $dmaster $others -# -# 8 -# +###################################################################### -echo verify that a recovery will revoke the delegations ... +echo "Verify that a recovery will revoke the delegations..." try_command_on_node 0 $CTDB recover -numreadonly=`try_command_on_node -v all $CTDB cattdb test.tdb | grep RO_HAVE | wc -l` -[ "$numreadonly" != "0" ] && { - echo BAD: recovery did not revoke delegations - exit 1 -} - -echo OK. test completed successfully -exit 0 +check_no_readonly diff --git a/ctdb/tests/src/ctdb_fetch_readonly_loop.c b/ctdb/tests/src/ctdb_fetch_readonly_loop.c index 5944fb7..bd171f5 100644 --- a/ctdb/tests/src/ctdb_fetch_readonly_loop.c +++ b/ctdb/tests/src/ctdb_fetch_readonly_loop.c @@ -45,7 +45,7 @@ static void fetch_lock_once(struct ctdb_context *ctdb, struct event_context *ev) // printf("Trying to fetch lock the record ...\n"); - h = ctdb_fetch_readonly_lock(ctdb_db, tmp_ctx, key, &data, true); + h = ctdb_fetch_readonly_lock(ctdb_db, tmp_ctx, key, &data, 1); if (h == NULL) { printf("Failed to fetch record '%s' on node %d\n", (const char *)key.dptr, ctdb_get_pnn(ctdb)); diff --git a/ctdb/tests/src/ctdb_update_record.c b/ctdb/tests/src/ctdb_update_record.c index 6eff1d0..78d983b 100644 --- a/ctdb/tests/src/ctdb_update_record.c +++ b/ctdb/tests/src/ctdb_update_record.c @@ -45,7 +45,7 @@ static void fetch_lock_once(struct ctdb_context *ctdb, struct event_context *ev, printf("Trying to fetch lock the record ...\n"); - h = ctdb_fetch_readonly_lock(ctdb_db, tmp_ctx, key, &data, false); + h = ctdb_fetch_lock(ctdb_db, tmp_ctx, key, &data); if (h == NULL) { printf("Failed to fetch record '%s' on node %d\n", (const char *)key.dptr, ctdb_get_pnn(ctdb)); -- Samba Shared Repository