accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject svn commit: r1444313 - in /accumulo/trunk: core/src/main/java/org/apache/accumulo/core/Constants.java server/src/main/java/org/apache/accumulo/server/master/Master.java test/system/upgrade_test.sh
Date Sat, 09 Feb 2013 02:10:15 GMT
Author: kturner
Date: Sat Feb  9 02:10:15 2013
New Revision: 1444313

URL: http://svn.apache.org/r1444313
Log:
ACCUMULO-635 ACCUMULO-1010 got basic upgrade and upgrade test script working... 

Modified:
    accumulo/trunk/core/src/main/java/org/apache/accumulo/core/Constants.java
    accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java
    accumulo/trunk/test/system/upgrade_test.sh

Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/Constants.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/Constants.java?rev=1444313&r1=1444312&r2=1444313&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/Constants.java (original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/Constants.java Sat Feb  9 02:10:15
2013
@@ -35,8 +35,8 @@ public class Constants {
   
   // versions should never be negative
   public static final Integer WIRE_VERSION = 2;
-  public static final int DATA_VERSION = 4;
-  public static final int PREV_DATA_VERSION = 3;
+  public static final int DATA_VERSION = 5;
+  public static final int PREV_DATA_VERSION = 4;
   
   // Zookeeper locations
   public static final String ZROOT = "/accumulo";

Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java?rev=1444313&r1=1444312&r2=1444313&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java Sat
Feb  9 02:10:15 2013
@@ -98,7 +98,6 @@ import org.apache.accumulo.fate.zookeepe
 import org.apache.accumulo.server.Accumulo;
 import org.apache.accumulo.server.client.HdfsZooInstance;
 import org.apache.accumulo.server.conf.ServerConfiguration;
-import org.apache.accumulo.server.iterators.MetadataBulkLoadFilter;
 import org.apache.accumulo.server.master.LiveTServerSet.TServerConnection;
 import org.apache.accumulo.server.master.balancer.DefaultLoadBalancer;
 import org.apache.accumulo.server.master.balancer.TabletBalancer;
@@ -157,7 +156,6 @@ import org.apache.accumulo.server.zookee
 import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
 import org.apache.accumulo.trace.instrument.thrift.TraceWrap;
 import org.apache.accumulo.trace.thrift.TInfo;
-import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.DataInputBuffer;
@@ -171,8 +169,6 @@ import org.apache.zookeeper.KeeperExcept
 import org.apache.zookeeper.KeeperException.NoNodeException;
 import org.apache.zookeeper.WatchedEvent;
 import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.ZooDefs;
-import org.apache.zookeeper.data.ACL;
 import org.apache.zookeeper.data.Stat;
 
 /**
@@ -279,53 +275,21 @@ public class Master implements LiveTServ
         log.info("Upgrading zookeeper");
         
         IZooReaderWriter zoo = ZooReaderWriter.getInstance();
-        
-        TablePropUtil.setTableProperty(Constants.METADATA_TABLE_ID, Property.TABLE_ITERATOR_PREFIX.getKey()
+ "majc.bulkLoadFilter", "20,"
-            + MetadataBulkLoadFilter.class.getName());
-        
-        zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZTABLE_LOCKS, new byte[0],
NodeExistsPolicy.SKIP);
-        zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZHDFS_RESERVATIONS, new
byte[0], NodeExistsPolicy.SKIP);
-        zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZNEXT_FILE, new byte[]
{'0'}, NodeExistsPolicy.SKIP);
-        
-        String[] tablePropsToDelete = new String[] {"table.scan.cache.size", "table.scan.cache.enable"};
-        
+
+        zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZRECOVERY, new byte[]
{'0'}, NodeExistsPolicy.SKIP);
+
         for (String id : Tables.getIdToNameMap(instance).keySet()) {
-          zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + id
+ Constants.ZTABLE_FLUSH_ID, "0".getBytes(), NodeExistsPolicy.SKIP);
-          zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + id
+ Constants.ZTABLE_COMPACT_ID, "0".getBytes(), NodeExistsPolicy.SKIP);
+
           zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + id
+ Constants.ZTABLE_COMPACT_CANCEL_ID, "0".getBytes(),
               NodeExistsPolicy.SKIP);
-          
-          for (String prop : tablePropsToDelete) {
-            String propPath = ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + id +
Constants.ZTABLE_CONF + "/" + prop;
-            if (zoo.exists(propPath))
-              zoo.delete(propPath, -1);
-          }
         }
-        
-        setACLs(zoo, ZooUtil.getRoot(instance), ZooUtil.getRoot(instance) + Constants.ZUSERS);
-        
       } catch (Exception ex) {
         log.fatal("Error performing upgrade", ex);
         System.exit(1);
       }
     }
   }
-  
-  private void setACLs(IZooReaderWriter zoo, String root, String users) throws Exception
{
-    Stat stat = new Stat();
-    List<ACL> acls = zoo.getZooKeeper().getACL(root, stat);
-    if (acls.equals(ZooDefs.Ids.OPEN_ACL_UNSAFE)) {
-      if (root.startsWith(users)) {
-        zoo.getZooKeeper().setACL(root, ZooUtil.PRIVATE, -1);
-      } else {
-        zoo.getZooKeeper().setACL(root, ZooUtil.PUBLIC, -1);
-      }
-      for (String child : zoo.getChildren(root)) {
-        setACLs(zoo, root + "/" + child, users);
-      }
-    }
-  }
-  
+
   private final AtomicBoolean upgradeMetadataRunning = new AtomicBoolean(false);
   
   private final ServerConfiguration serverConfig;
@@ -337,22 +301,6 @@ public class Master implements LiveTServ
           @Override
           public void run() {
             try {
-              // add delete entries to metadata table for bulk dirs
-              
-              log.info("Adding bulk dir delete entries to !METADATA table for upgrade");
-              
-              BatchWriter bw = getConnector().createBatchWriter(Constants.METADATA_TABLE_NAME,
new BatchWriterConfig());
-              
-              FileStatus[] tables = fs.globStatus(new Path(Constants.getTablesDir(getSystemConfiguration())
+ "/*"));
-              for (FileStatus tableDir : tables) {
-                FileStatus[] bulkDirs = fs.globStatus(new Path(tableDir.getPath() + "/bulk_*"));
-                for (FileStatus bulkDir : bulkDirs) {
-                  bw.addMutation(MetadataTable.createDeleteMutation(tableDir.getPath().getName(),
"/" + bulkDir.getPath().getName()));
-                }
-              }
-              
-              bw.close();
-              
               Accumulo.updateAccumuloVersion(fs);
               
               log.info("Upgrade complete");

Modified: accumulo/trunk/test/system/upgrade_test.sh
URL: http://svn.apache.org/viewvc/accumulo/trunk/test/system/upgrade_test.sh?rev=1444313&r1=1444312&r2=1444313&view=diff
==============================================================================
--- accumulo/trunk/test/system/upgrade_test.sh (original)
+++ accumulo/trunk/test/system/upgrade_test.sh Sat Feb  9 02:10:15 2013
@@ -22,59 +22,53 @@ then
   exit -1
 fi
 
-#this script test upgrade from 1.3 to 1.4.  This script is not self verifying, its output
must be inspected for correctness.
+#this script test upgrade from 1.4 to 1.5.  This script is not self verifying, its output
must be inspected for correctness.
 
-#set the following to point to configured 1.3 and 1.4 accumulo dirs.  Ensure both point to
the same walogs
+#set the following to point to configured 1.4 and 1.5 accumulo dirs.  Ensure both point to
the same walogs
 
 #TODO could support multinode configs, this script assumes single node config
 
-ONE_THREE_DIR=../../../accumulo-1.3.5-SNAPSHOT
-ONE_FOUR_DIR=../../
+ONE_FOUR_DIR=../../../accumulo-1.4
+ONE_FIVE_DIR=../../
 
 pkill -f accumulo.start
 hadoop fs -rmr /accumulo
 hadoop fs -rmr /testmf
 hadoop fs -rmr /testmfFail
 
-echo "uptest\nsecret\nsecret" | $ONE_THREE_DIR/bin/accumulo init --clear-instance-name
-$ONE_THREE_DIR/bin/start-all.sh
-$ONE_THREE_DIR/bin/accumulo org.apache.accumulo.test.TestIngest -u root -p secret --timestamp
1 --size 50 --random 56 --rows 100000 --start 0 --cols 1
-$ONE_THREE_DIR/bin/accumulo org.apache.accumulo.test.TestIngest --rfile /testmf/mf01 --timestamp
1 --size 50 --random 56 --rows 100000 --start 100000 --cols 1
-$ONE_THREE_DIR/bin/accumulo org.apache.accumulo.test.BulkImportDirectory -u root -p secret
-t test_ingest --source /testmf --failures /testmfFail
+echo -e "uptest\nsecret\nsecret" | $ONE_FOUR_DIR/bin/accumulo init --clear-instance-name
+$ONE_FOUR_DIR/bin/start-all.sh
+$ONE_FOUR_DIR/bin/accumulo 'org.apache.accumulo.server.test.TestIngest$CreateTable' 0 200000
10 root secret
+$ONE_FOUR_DIR/bin/accumulo org.apache.accumulo.server.test.TestIngest -timestamp 1 -size
50 -random 56 100000 0 1
+$ONE_FOUR_DIR/bin/accumulo org.apache.accumulo.server.test.TestIngest -rfile /testmf/mf01
-timestamp 1 -size 50 -random 56 100000 100000 1
+$ONE_FOUR_DIR/bin/accumulo org.apache.accumulo.server.test.BulkImportDirectory root secret
test_ingest /testmf /testmfFail
 if [ $1 == "dirty" ]; then
 	pkill -9 -f accumulo.start
 else 
-	$ONE_THREE_DIR/bin/stop-all.sh
+	$ONE_FOUR_DIR/bin/stop-all.sh
 fi
 
-echo "==== Starting 1.4 ==="
+echo "==== Starting 1.5 ==="
 
-#Test some new 1.4 features against an upgraded instance
-#TODO need to try following operations in different orders
-#TODO test delete range
 
-$ONE_FOUR_DIR/bin/start-all.sh
-$ONE_FOUR_DIR/bin/accumulo  org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 1
--random 56 --rows 200000 --start 0 --cols 1 
-echo "compact -t test_ingest -w" | $ONE_FOUR_DIR/bin/accumulo shell -u root -p secret
-$ONE_FOUR_DIR/bin/accumulo  org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 1
--random 56 --rows 200000 --start 0 --cols 1
-echo "merge -t test_ingest -s 1G" | $ONE_FOUR_DIR/bin/accumulo shell -u root -p secret
-$ONE_FOUR_DIR/bin/accumulo  org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 1
--random 56 --rows 200000 --start 0 --cols 1
-echo "clonetable test_ingest tmp\ndeletetable test_ingest\nrenametable tmp test_ingest" |
$ONE_FOUR_DIR/bin/accumulo shell -u root -p secret
-$ONE_FOUR_DIR/bin/accumulo  org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 1
--random 56 --rows 200000 --start 0 --cols 1
-
-#test overwriting data writting in 1.3
-$ONE_FOUR_DIR/bin/accumulo org.apache.accumulo.test.TestIngest --timestamp 2 --size 50 --random
57 --rows 300000 --start 0 --cols 1
-$ONE_FOUR_DIR/bin/accumulo  org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2
--random 57 --rows 300000 --start 0 --cols 1
-echo "compact -t test_ingest -w" | $ONE_FOUR_DIR/bin/accumulo shell -u root -p secret
-$ONE_FOUR_DIR/bin/accumulo  org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2
--random 57 --rows 300000 --start 0 --cols 1
+$ONE_FIVE_DIR/bin/start-all.sh
+$ONE_FIVE_DIR/bin/accumulo  org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 1
--random 56 --rows 200000 --start 0 --cols 1 
+echo "compact -t test_ingest -w" | $ONE_FIVE_DIR/bin/accumulo shell -u root -p secret
+$ONE_FIVE_DIR/bin/accumulo  org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 1
--random 56 --rows 200000 --start 0 --cols 1
 
-$ONE_FOUR_DIR/bin/stop-all.sh
-$ONE_FOUR_DIR/bin/start-all.sh
+#test overwriting data writting in 1.4
+$ONE_FIVE_DIR/bin/accumulo org.apache.accumulo.test.TestIngest --timestamp 2 --size 50 --random
57 --rows 300000 --start 0 --cols 1 -cv "L1&L2&G1&GROUP2"
+$ONE_FIVE_DIR/bin/accumulo  org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2
--random 57 --rows 300000 --start 0 --cols 1
+echo "compact -t test_ingest -w" | $ONE_FIVE_DIR/bin/accumulo shell -u root -p secret
+$ONE_FIVE_DIR/bin/accumulo  org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2
--random 57 --rows 300000 --start 0 --cols 1
 
-$ONE_FOUR_DIR/bin/accumulo  org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2
--random 57 --rows 300000 --start 0 --cols 1
+$ONE_FIVE_DIR/bin/stop-all.sh
+$ONE_FIVE_DIR/bin/start-all.sh
 
-pkill -f accumulo.start
-$ONE_FOUR_DIR/bin/start-all.sh
+$ONE_FIVE_DIR/bin/accumulo  org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2
--random 57 --rows 300000 --start 0 --cols 1
+
+pkill -9 -f accumulo.start
+$ONE_FIVE_DIR/bin/start-all.sh
 
-$ONE_FOUR_DIR/bin/accumulo  org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2
--random 57 --rows 300000 --start 0 --cols 1
+$ONE_FIVE_DIR/bin/accumulo  org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2
--random 57 --rows 300000 --start 0 --cols 1
 



Mime
View raw message