accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject svn commit: r1204788 - in /incubator/accumulo/trunk: ./ src/core/src/main/java/org/apache/accumulo/core/ src/server/src/main/java/org/apache/accumulo/server/ src/server/src/main/java/org/apache/accumulo/server/master/ src/server/src/main/java/org/apach...
Date Tue, 22 Nov 2011 02:36:13 GMT
Author: kturner
Date: Tue Nov 22 02:36:13 2011
New Revision: 1204788

URL: http://svn.apache.org/viewvc?rev=1204788&view=rev
Log:
ACCUMULO-169 merge upgrade code to trunk

Added:
    incubator/accumulo/trunk/test/system/upgrade_test.sh
      - copied unchanged from r1204786, incubator/accumulo/branches/1.4/test/system/upgrade_test.sh
Removed:
    incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/upgrade/
Modified:
    incubator/accumulo/trunk/   (props changed)
    incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/Constants.java
    incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java
    incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
    incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java

Propchange: incubator/accumulo/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Nov 22 02:36:13 2011
@@ -1,2 +1,2 @@
 /incubator/accumulo/branches/1.3:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625
-/incubator/accumulo/branches/1.4:1201902-1204760
+/incubator/accumulo/branches/1.4:1201902-1204786

Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/Constants.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/Constants.java?rev=1204788&r1=1204787&r2=1204788&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/Constants.java
(original)
+++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/Constants.java
Tue Nov 22 02:36:13 2011
@@ -29,7 +29,8 @@ import org.apache.hadoop.io.Text;
 
 public class Constants {
   public static final String VERSION = "1.5.0-incubating-SNAPSHOT";
-  public static final int DATA_VERSION = 3;
+  public static final int DATA_VERSION = 4;
+  public static final int PREV_DATA_VERSION = 3;
   
   // Zookeeper locations
   public static final String ZROOT = "/accumulo";

Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java?rev=1204788&r1=1204787&r2=1204788&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java
(original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java
Tue Nov 22 02:36:13 2011
@@ -36,6 +36,7 @@ import org.apache.accumulo.server.zookee
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction;
 import org.apache.log4j.Logger;
@@ -48,6 +49,23 @@ public class Accumulo {
   private static final Logger log = Logger.getLogger(Accumulo.class);
   private static Integer dataVersion = null;
   
+  public static synchronized void updateAccumuloVersion() {
+    Configuration conf = CachedConfiguration.getInstance();
+    try {
+      if (getAccumuloPersistentVersion() == Constants.PREV_DATA_VERSION) {
+        FileSystem fs = TraceFileSystem.wrap(FileUtil.getFileSystem(conf, ServerConfiguration.getSiteConfiguration()));
+        
+        fs.create(new Path(ServerConstants.getDataVersionLocation() + "/" + Constants.DATA_VERSION));
+        fs.delete(new Path(ServerConstants.getDataVersionLocation() + "/" + Constants.PREV_DATA_VERSION),
false);
+        
+        dataVersion = null;
+      }
+    } catch (IOException e) {
+      throw new RuntimeException("Unable to set accumulo version: an error occurred.", e);
+    }
+    
+  }
+
   public static synchronized int getAccumuloPersistentVersion() {
     if (dataVersion != null)
       return dataVersion;
@@ -113,7 +131,7 @@ public class Accumulo {
     
     int dataVersion = Accumulo.getAccumuloPersistentVersion();
     Version codeVersion = new Version(Constants.VERSION);
-    if (dataVersion != Constants.DATA_VERSION) {
+    if (dataVersion != Constants.DATA_VERSION && dataVersion != Constants.PREV_DATA_VERSION)
{
       throw new RuntimeException("This version of accumulo (" + codeVersion + ") is not compatible
with files stored using data version " + dataVersion);
     }
     

Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/Master.java?rev=1204788&r1=1204787&r2=1204788&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
(original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
Tue Nov 22 02:36:13 2011
@@ -35,6 +35,7 @@ import java.util.SortedMap;
 import java.util.TimerTask;
 import java.util.TreeMap;
 import java.util.TreeSet;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.accumulo.core.Constants;
@@ -52,7 +53,6 @@ import org.apache.accumulo.core.client.i
 import org.apache.accumulo.core.client.impl.thrift.TableOperation;
 import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType;
 import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
 import org.apache.accumulo.core.conf.Property;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.KeyExtent;
@@ -143,7 +143,6 @@ import org.apache.accumulo.server.util.A
 import org.apache.accumulo.server.util.DefaultMap;
 import org.apache.accumulo.server.util.Halt;
 import org.apache.accumulo.server.util.MetadataTable;
-import org.apache.accumulo.server.util.OfflineMetadataScanner;
 import org.apache.accumulo.server.util.SystemPropUtil;
 import org.apache.accumulo.server.util.TServerUtils;
 import org.apache.accumulo.server.util.TablePropUtil;
@@ -156,6 +155,7 @@ import org.apache.accumulo.server.zookee
 import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 import org.apache.accumulo.server.zookeeper.ZooReaderWriter.Mutator;
 import org.apache.accumulo.start.classloader.AccumuloClassLoader;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.DataInputBuffer;
@@ -308,62 +308,85 @@ public class Master implements LiveTServ
       }, 100l, 1000l);
     }
     
-    if (oldState != newState && (newState == MasterState.SAFE_MODE || newState ==
MasterState.NORMAL)) {
-      upgradeSettings();
+    if (oldState != newState && (newState == MasterState.HAVE_LOCK)) {
+      upgradeZookeeper();
+    }
+    
+    if (oldState != newState && (newState == MasterState.NORMAL)) {
+      upgradeMetadata();
     }
   }
   
-  private void upgradeSettings() {
-    AccumuloConfiguration conf = ServerConfiguration.getTableConfiguration(Constants.METADATA_TABLE_ID);
-    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
-    if (!conf.getBoolean(Property.TABLE_BLOCKCACHE_ENABLED)) {
+  private void upgradeZookeeper() {
+    if (Accumulo.getAccumuloPersistentVersion() == Constants.PREV_DATA_VERSION) {
+      // TODO check if tablets are loaded, if so abort?
+      
       try {
-        // make sure the last shutdown was clean
-        OfflineMetadataScanner scanner = new OfflineMetadataScanner();
-        scanner.fetchColumnFamily(Constants.METADATA_LOG_COLUMN_FAMILY);
-        boolean fail = false;
-        for (Entry<Key,Value> entry : scanner) {
-          log.error(String.format("Unable to upgrade: extent %s has log entry %s", entry.getKey().getRow(),
entry.getValue()));
-          fail = true;
-        }
-        if (fail)
-          throw new Exception("Upgrade requires a clean shutdown");
+        // TODO compare zookeeper dump of 1.3 and 1.4 zookeeper init to make sure no more
zookeeper updates are needed
+        // TODO need to update ACL in zookeeper
+        log.info("Upgrading zookeeper");
+
+        IZooReaderWriter zoo = ZooReaderWriter.getInstance();
         
-        // perform 1.2 -> 1.3 settings
-        zset(Property.TABLE_LOCALITY_GROUP_PREFIX.getKey() + "tablet",
-            String.format("%s,%s", Constants.METADATA_TABLET_COLUMN_FAMILY.toString(), Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY.toString()));
-        zset(Property.TABLE_LOCALITY_GROUP_PREFIX.getKey() + "server", String.format("%s,%s,%s,%s",
Constants.METADATA_DATAFILE_COLUMN_FAMILY.toString(),
-            Constants.METADATA_LOG_COLUMN_FAMILY.toString(), Constants.METADATA_SERVER_COLUMN_FAMILY.toString(),
-            Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY.toString()));
-        zset(Property.TABLE_LOCALITY_GROUPS.getKey(), "tablet,server");
-        zset(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY.getKey(), "");
-        zset(Property.TABLE_INDEXCACHE_ENABLED.getKey(), "true");
-        zset(Property.TABLE_BLOCKCACHE_ENABLED.getKey(), "true");
-        for (String id : Tables.getIdToNameMap(instance).keySet())
-          zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + id
+ "/state", "ONLINE".getBytes(), NodeExistsPolicy.OVERWRITE);
+        zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZTABLE_LOCKS, new byte[0],
NodeExistsPolicy.SKIP);
+        zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZHDFS_RESERVATIONS, new
byte[0], NodeExistsPolicy.SKIP);
+        zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZNEXT_FILE, new byte[]
{'0'}, NodeExistsPolicy.SKIP);
+
+        for (String id : Tables.getIdToNameMap(instance).keySet()) {
+          zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + id
+ Constants.ZTABLE_FLUSH_ID, "0".getBytes(), NodeExistsPolicy.SKIP);
+          zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + id
+ Constants.ZTABLE_COMPACT_ID, "0".getBytes(), NodeExistsPolicy.SKIP);
+        }
       } catch (Exception ex) {
         log.fatal("Error performing upgrade", ex);
         System.exit(1);
       }
-      
     }
-    String zkInstanceRoot = ZooUtil.getRoot(instance);
-    try {
-      if (!zoo.exists(zkInstanceRoot + Constants.ZTABLE_LOCKS)) {
-        zoo.putPersistentData(zkInstanceRoot + Constants.ZTABLE_LOCKS, new byte[0], NodeExistsPolicy.FAIL);
-        zoo.putPersistentData(zkInstanceRoot + Constants.ZHDFS_RESERVATIONS, new byte[0],
NodeExistsPolicy.FAIL);
-      }
-    } catch (Exception ex) {
-      log.fatal("Error performing upgrade", ex);
-      System.exit(1);
-    }
-    
   }
   
-  private static void zset(String property, String value) throws KeeperException, InterruptedException
{
-    TablePropUtil.setTableProperty(Constants.METADATA_TABLE_ID, property, value);
+  private AtomicBoolean upgradeMetadataRunning = new AtomicBoolean(false);
+
+  private void upgradeMetadata() {
+    if (Accumulo.getAccumuloPersistentVersion() == Constants.PREV_DATA_VERSION) {
+      if (upgradeMetadataRunning.compareAndSet(false, true)) {
+        Runnable upgradeTask = new Runnable() {
+          @Override
+          public void run() {
+            try {
+              // add delete entries to metadata table for bulk dirs
+              
+              log.info("Adding bulk dir delete entries to !METADATA table for upgrade");
+
+              BatchWriter bw = getConnector().createBatchWriter(Constants.METADATA_TABLE_NAME,
10000000, 60000l, 4);
+              
+              FileStatus[] tables = fs.globStatus(new Path(Constants.getTablesDir(ServerConfiguration.getSystemConfiguration())
+ "/*"));
+              for (FileStatus tableDir : tables) {
+                FileStatus[] bulkDirs = fs.globStatus(new Path(tableDir.getPath() + "/bulk_*"));
+                for (FileStatus bulkDir : bulkDirs) {
+                  bw.addMutation(MetadataTable.createDeleteMutation(tableDir.getPath().getName(),
"/" + bulkDir.getPath().getName()));
+                }
+              }
+              
+              bw.close();
+              
+              Accumulo.updateAccumuloVersion();
+              
+              log.info("Upgrade complete");
+
+            } catch (Exception ex) {
+              log.fatal("Error performing upgrade", ex);
+              System.exit(1);
+            }
+            
+          }
+        };
+        
+        // need to run this in a separate thread because a lock is held that prevents !METADATA
tablets from being assigned and this task writes to the
+        // !METADATA table
+        new Thread(upgradeTask).start();
+      }
+    }
   }
-  
+
   private int assignedOrHosted(Text tableId) {
     int result = 0;
     for (TabletGroupWatcher watcher : watchers) {

Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java?rev=1204788&r1=1204787&r2=1204788&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
(original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
Tue Nov 22 02:36:13 2011
@@ -525,7 +525,6 @@ public class MetadataTable extends org.a
   
   public static Mutation createDeleteMutation(String tableId, String pathToRemove) {
     Mutation delFlag;
-    ;
     if (pathToRemove.startsWith("../"))
       delFlag = new Mutation(new Text(Constants.METADATA_DELETE_FLAG_PREFIX + pathToRemove.substring(2)));
     else



Mime
View raw message