accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject [1/3] accumulo git commit: ACCUMULO-4324 Fixed issues found while attempting to run upgrade test
Date Fri, 22 Jul 2016 20:33:36 GMT
Repository: accumulo
Updated Branches:
  refs/heads/master 7c1d58a41 -> 1ac95b6c6


ACCUMULO-4324 Fixed issues found while attempting to run upgrade test

With these fixes was able to run the update test script going from 1.7.1 and
1.6.5 to 1.8.0-SNAP.  Ran script with clean and dirty shutdown.

Also removed option to upgrade from 1.5 and ealier since this was not tested.

Closes apache/accumulo#120, apache/accumulo#121


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a11b28b3
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a11b28b3
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a11b28b3

Branch: refs/heads/master
Commit: a11b28b356011acabc46c23393621012a080fac9
Parents: c8aa1fc
Author: Keith Turner <kturner@apache.org>
Authored: Tue Jul 12 17:24:24 2016 -0400
Committer: Keith Turner <kturner@apache.org>
Committed: Fri Jul 22 16:15:39 2016 -0400

----------------------------------------------------------------------
 .../apache/accumulo/server/ServerConstants.java |  2 +-
 .../accumulo/server/log/WalStateManager.java    | 19 +++++++++++++--
 .../java/org/apache/accumulo/master/Master.java |  4 ++++
 test/system/upgrade_test.sh                     | 25 ++++++++++----------
 4 files changed, 35 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/a11b28b3/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
index e840ea2..a14b8fc 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
@@ -72,7 +72,7 @@ public class ServerConstants {
   public static final int LOGGING_TO_HDFS = 4;
   public static final BitSet CAN_UPGRADE = new BitSet();
   static {
-    for (int i : new int[] {DATA_VERSION, MOVE_TO_ROOT_TABLE, MOVE_DELETE_MARKERS, LOGGING_TO_HDFS})
{
+    for (int i : new int[] {DATA_VERSION, MOVE_TO_REPLICATION_TABLE, MOVE_TO_ROOT_TABLE})
{
       CAN_UPGRADE.set(i);
     }
   }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/a11b28b3/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java
b/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java
index 25d9168..f08bcc4 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java
@@ -86,18 +86,33 @@ public class WalStateManager {
   private final Instance instance;
   private final ZooReaderWriter zoo;
 
+  private volatile boolean checkedExistance = false;
+
   public WalStateManager(Instance instance, ZooReaderWriter zoo) {
     this.instance = instance;
     this.zoo = zoo;
   }
 
-  private String root() {
-    return ZooUtil.getRoot(instance) + ZWALS;
+  private String root() throws WalMarkerException {
+    String root = ZooUtil.getRoot(instance) + ZWALS;
+
+    try {
+      if (!checkedExistance && !zoo.exists(root)) {
+        zoo.putPersistentData(root, new byte[0], NodeExistsPolicy.SKIP);
+      }
+
+      checkedExistance = true;
+    } catch (KeeperException | InterruptedException e) {
+      throw new WalMarkerException(e);
+    }
+
+    return root;
   }
 
   // Tablet server exists
   public void initWalMarker(TServerInstance tsi) throws WalMarkerException {
     byte[] data = new byte[0];
+
     try {
       zoo.putPersistentData(root() + "/" + tsi.toString(), data, NodeExistsPolicy.FAIL);
     } catch (KeeperException | InterruptedException e) {

http://git-wip-us.apache.org/repos/asf/accumulo/blob/a11b28b3/server/master/src/main/java/org/apache/accumulo/master/Master.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/Master.java b/server/master/src/main/java/org/apache/accumulo/master/Master.java
index 76fafb5..9633a9d 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/Master.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java
@@ -433,6 +433,10 @@ public class Master extends AccumuloServerContext implements LiveTServerSet.List
 
         // add the currlog location for root tablet current logs
         zoo.putPersistentData(ZooUtil.getRoot(getInstance()) + RootTable.ZROOT_TABLET_CURRENT_LOGS,
new byte[0], NodeExistsPolicy.SKIP);
+
+        // create tablet server wal logs node in ZK
+        zoo.putPersistentData(ZooUtil.getRoot(getInstance()) + WalStateManager.ZWALS, new
byte[0], NodeExistsPolicy.SKIP);
+
         haveUpgradedZooKeeper = true;
       } catch (Exception ex) {
         // ACCUMULO-3651 Changed level to error and added FATAL to message for slf4j compatibility

http://git-wip-us.apache.org/repos/asf/accumulo/blob/a11b28b3/test/system/upgrade_test.sh
----------------------------------------------------------------------
diff --git a/test/system/upgrade_test.sh b/test/system/upgrade_test.sh
index 590c07c..651755d 100755
--- a/test/system/upgrade_test.sh
+++ b/test/system/upgrade_test.sh
@@ -28,21 +28,22 @@ fi
 
 #TODO could support multinode configs, this script assumes single node config
 
-PREV=../../../../accumulo-1.5.0
+PREV=../../../accumulo-1.7.1
 CURR=../../
 DIR=/accumulo
 BULK=/tmp/upt
+INSTANCE=testUp
 
 pkill -f accumulo.start
 hadoop fs -rmr "$DIR"
 hadoop fs -rmr "$BULK"
-hadoop fs -mkdir "$BULK/fail"
+hadoop fs -mkdir -p "$BULK/fail"
 
-"$PREV/bin/accumulo" init --clear-instance-name --instance-name testUp --password secret
+"$PREV/bin/accumulo" init --clear-instance-name --instance-name $INSTANCE --password secret
 "$PREV/bin/start-all.sh"
 
-"$PREV/bin/accumulo" org.apache.accumulo.test.TestIngest -u root -p secret --timestamp 1
--size 50 --random 56 --rows 200000 --start 0 --cols 1  --createTable --splits 10
-"$PREV/bin/accumulo" org.apache.accumulo.test.TestIngest --rfile $BULK/bulk/test --timestamp
1 --size 50 --random 56 --rows 200000 --start 200000 --cols 1
+"$PREV/bin/accumulo" org.apache.accumulo.test.TestIngest -i $INSTANCE -u root -p secret --timestamp
1 --size 50 --random 56 --rows 200000 --start 0 --cols 1  --createTable --splits 10
+"$PREV/bin/accumulo" org.apache.accumulo.test.TestIngest -i $INSTANCE -u root -p secret --rfile
$BULK/bulk/test --timestamp 1 --size 50 --random 56 --rows 200000 --start 200000 --cols 1
 
 echo -e "table test_ingest\nimportdirectory $BULK/bulk $BULK/fail false" | $PREV/bin/accumulo
shell -u root -p secret
 if [[ $1 == dirty ]]; then
@@ -54,23 +55,23 @@ fi
 echo "==== Starting Current ==="
 
 "$CURR/bin/start-all.sh"
-"$CURR/bin/accumulo" org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 1 --random
56 --rows 400000 --start 0 --cols 1 -u root -p secret
+"$CURR/bin/accumulo" org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 1 --random
56 --rows 400000 --start 0 --cols 1 -i $INSTANCE -u root -p secret
 echo "compact -t test_ingest -w" | $CURR/bin/accumulo shell -u root -p secret
-"$CURR/bin/accumulo" org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 1 --random
56 --rows 400000 --start 0 --cols 1 -u root -p secret
+"$CURR/bin/accumulo" org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 1 --random
56 --rows 400000 --start 0 --cols 1 -i $INSTANCE -u root -p secret
 
 
-"$CURR/bin/accumulo" org.apache.accumulo.test.TestIngest --timestamp 2 --size 50 --random
57 --rows 500000 --start 0 --cols 1 -u root -p secret
-"$CURR/bin/accumulo" org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2 --random
57 --rows 500000 --start 0 --cols 1 -u root -p secret
+"$CURR/bin/accumulo" org.apache.accumulo.test.TestIngest --timestamp 2 --size 50 --random
57 --rows 500000 --start 0 --cols 1 -i $INSTANCE -u root -p secret
+"$CURR/bin/accumulo" org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2 --random
57 --rows 500000 --start 0 --cols 1 -i $INSTANCE -u root -p secret
 echo "compact -t test_ingest -w" | $CURR/bin/accumulo shell -u root -p secret
-"$CURR/bin/accumulo" org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2 --random
57 --rows 500000 --start 0 --cols 1 -u root -p secret
+"$CURR/bin/accumulo" org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2 --random
57 --rows 500000 --start 0 --cols 1 -i $INSTANCE -u root -p secret
 
 "$CURR/bin/stop-all.sh"
 "$CURR/bin/start-all.sh"
 
-"$CURR/bin/accumulo" org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2 --random
57 --rows 500000 --start 0 --cols 1 -u root -p secret
+"$CURR/bin/accumulo" org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2 --random
57 --rows 500000 --start 0 --cols 1 -i $INSTANCE -u root -p secret
 
 pkill -9 -f accumulo.start
 "$CURR/bin/start-all.sh"
 
-"$CURR/bin/accumulo" org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2 --random
57 --rows 500000 --start 0 --cols 1 -u root -p secret
+"$CURR/bin/accumulo" org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2 --random
57 --rows 500000 --start 0 --cols 1 -i $INSTANCE -u root -p secret
 


Mime
View raw message