Repository: accumulo Updated Branches: refs/heads/master 7c1d58a41 -> 1ac95b6c6
ACCUMULO-4324 Fixed issues found while attempting to run upgrade test With these fixes was able to run the update test script going from 1.7.1 and 1.6.5 to 1.8.0-SNAP. Ran script with clean and dirty shutdown. Also removed option to upgrade from 1.5 and ealier since this was not tested. Closes apache/accumulo#120, apache/accumulo#121 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a11b28b3 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a11b28b3 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a11b28b3 Branch: refs/heads/master Commit: a11b28b356011acabc46c23393621012a080fac9 Parents: c8aa1fc Author: Keith Turner <ktur...@apache.org> Authored: Tue Jul 12 17:24:24 2016 -0400 Committer: Keith Turner <ktur...@apache.org> Committed: Fri Jul 22 16:15:39 2016 -0400 ---------------------------------------------------------------------- .../apache/accumulo/server/ServerConstants.java | 2 +- .../accumulo/server/log/WalStateManager.java | 19 +++++++++++++-- .../java/org/apache/accumulo/master/Master.java | 4 ++++ test/system/upgrade_test.sh | 25 ++++++++++---------- 4 files changed, 35 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/a11b28b3/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java index e840ea2..a14b8fc 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java +++ b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java @@ -72,7 +72,7 @@ public class ServerConstants { public static final int LOGGING_TO_HDFS = 4; public static final BitSet CAN_UPGRADE = new BitSet(); static { - for (int i : new int[] {DATA_VERSION, MOVE_TO_ROOT_TABLE, MOVE_DELETE_MARKERS, LOGGING_TO_HDFS}) { + for (int i : new int[] {DATA_VERSION, MOVE_TO_REPLICATION_TABLE, MOVE_TO_ROOT_TABLE}) { CAN_UPGRADE.set(i); } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/a11b28b3/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java b/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java index 25d9168..f08bcc4 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java +++ b/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java @@ -86,18 +86,33 @@ public class WalStateManager { private final Instance instance; private final ZooReaderWriter zoo; + private volatile boolean checkedExistance = false; + public WalStateManager(Instance instance, ZooReaderWriter zoo) { this.instance = instance; this.zoo = zoo; } - private String root() { - return ZooUtil.getRoot(instance) + ZWALS; + private String root() throws WalMarkerException { + String root = ZooUtil.getRoot(instance) + ZWALS; + + try { + if (!checkedExistance && !zoo.exists(root)) { + zoo.putPersistentData(root, new byte[0], NodeExistsPolicy.SKIP); + } + + checkedExistance = true; + } catch (KeeperException | InterruptedException e) { + throw new WalMarkerException(e); + } + + return root; } // Tablet server exists public void initWalMarker(TServerInstance tsi) throws WalMarkerException { byte[] data = new byte[0]; + try { zoo.putPersistentData(root() + "/" + tsi.toString(), data, NodeExistsPolicy.FAIL); } catch (KeeperException | InterruptedException e) { http://git-wip-us.apache.org/repos/asf/accumulo/blob/a11b28b3/server/master/src/main/java/org/apache/accumulo/master/Master.java ---------------------------------------------------------------------- diff --git a/server/master/src/main/java/org/apache/accumulo/master/Master.java b/server/master/src/main/java/org/apache/accumulo/master/Master.java index 76fafb5..9633a9d 100644 --- a/server/master/src/main/java/org/apache/accumulo/master/Master.java +++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java @@ -433,6 +433,10 @@ public class Master extends AccumuloServerContext implements LiveTServerSet.List // add the currlog location for root tablet current logs zoo.putPersistentData(ZooUtil.getRoot(getInstance()) + RootTable.ZROOT_TABLET_CURRENT_LOGS, new byte[0], NodeExistsPolicy.SKIP); + + // create tablet server wal logs node in ZK + zoo.putPersistentData(ZooUtil.getRoot(getInstance()) + WalStateManager.ZWALS, new byte[0], NodeExistsPolicy.SKIP); + haveUpgradedZooKeeper = true; } catch (Exception ex) { // ACCUMULO-3651 Changed level to error and added FATAL to message for slf4j compatibility http://git-wip-us.apache.org/repos/asf/accumulo/blob/a11b28b3/test/system/upgrade_test.sh ---------------------------------------------------------------------- diff --git a/test/system/upgrade_test.sh b/test/system/upgrade_test.sh index 590c07c..651755d 100755 --- a/test/system/upgrade_test.sh +++ b/test/system/upgrade_test.sh @@ -28,21 +28,22 @@ fi #TODO could support multinode configs, this script assumes single node config -PREV=../../../../accumulo-1.5.0 +PREV=../../../accumulo-1.7.1 CURR=../../ DIR=/accumulo BULK=/tmp/upt +INSTANCE=testUp pkill -f accumulo.start hadoop fs -rmr "$DIR" hadoop fs -rmr "$BULK" -hadoop fs -mkdir "$BULK/fail" +hadoop fs -mkdir -p "$BULK/fail" -"$PREV/bin/accumulo" init --clear-instance-name --instance-name testUp --password secret +"$PREV/bin/accumulo" init --clear-instance-name --instance-name $INSTANCE --password secret "$PREV/bin/start-all.sh" -"$PREV/bin/accumulo" org.apache.accumulo.test.TestIngest -u root -p secret --timestamp 1 --size 50 --random 56 --rows 200000 --start 0 --cols 1 --createTable --splits 10 -"$PREV/bin/accumulo" org.apache.accumulo.test.TestIngest --rfile $BULK/bulk/test --timestamp 1 --size 50 --random 56 --rows 200000 --start 200000 --cols 1 +"$PREV/bin/accumulo" org.apache.accumulo.test.TestIngest -i $INSTANCE -u root -p secret --timestamp 1 --size 50 --random 56 --rows 200000 --start 0 --cols 1 --createTable --splits 10 +"$PREV/bin/accumulo" org.apache.accumulo.test.TestIngest -i $INSTANCE -u root -p secret --rfile $BULK/bulk/test --timestamp 1 --size 50 --random 56 --rows 200000 --start 200000 --cols 1 echo -e "table test_ingest\nimportdirectory $BULK/bulk $BULK/fail false" | $PREV/bin/accumulo shell -u root -p secret if [[ $1 == dirty ]]; then @@ -54,23 +55,23 @@ fi echo "==== Starting Current ===" "$CURR/bin/start-all.sh" -"$CURR/bin/accumulo" org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 1 --random 56 --rows 400000 --start 0 --cols 1 -u root -p secret +"$CURR/bin/accumulo" org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 1 --random 56 --rows 400000 --start 0 --cols 1 -i $INSTANCE -u root -p secret echo "compact -t test_ingest -w" | $CURR/bin/accumulo shell -u root -p secret -"$CURR/bin/accumulo" org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 1 --random 56 --rows 400000 --start 0 --cols 1 -u root -p secret +"$CURR/bin/accumulo" org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 1 --random 56 --rows 400000 --start 0 --cols 1 -i $INSTANCE -u root -p secret -"$CURR/bin/accumulo" org.apache.accumulo.test.TestIngest --timestamp 2 --size 50 --random 57 --rows 500000 --start 0 --cols 1 -u root -p secret -"$CURR/bin/accumulo" org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2 --random 57 --rows 500000 --start 0 --cols 1 -u root -p secret +"$CURR/bin/accumulo" org.apache.accumulo.test.TestIngest --timestamp 2 --size 50 --random 57 --rows 500000 --start 0 --cols 1 -i $INSTANCE -u root -p secret +"$CURR/bin/accumulo" org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2 --random 57 --rows 500000 --start 0 --cols 1 -i $INSTANCE -u root -p secret echo "compact -t test_ingest -w" | $CURR/bin/accumulo shell -u root -p secret -"$CURR/bin/accumulo" org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2 --random 57 --rows 500000 --start 0 --cols 1 -u root -p secret +"$CURR/bin/accumulo" org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2 --random 57 --rows 500000 --start 0 --cols 1 -i $INSTANCE -u root -p secret "$CURR/bin/stop-all.sh" "$CURR/bin/start-all.sh" -"$CURR/bin/accumulo" org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2 --random 57 --rows 500000 --start 0 --cols 1 -u root -p secret +"$CURR/bin/accumulo" org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2 --random 57 --rows 500000 --start 0 --cols 1 -i $INSTANCE -u root -p secret pkill -9 -f accumulo.start "$CURR/bin/start-all.sh" -"$CURR/bin/accumulo" org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2 --random 57 --rows 500000 --start 0 --cols 1 -u root -p secret +"$CURR/bin/accumulo" org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2 --random 57 --rows 500000 --start 0 --cols 1 -i $INSTANCE -u root -p secret
