accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject git commit: ACCUMULO-1481 creates root table and copies files to the directory on master start-up
Date Mon, 28 Oct 2013 21:15:02 GMT
Updated Branches:
  refs/heads/master 8683295fc -> e04fd1902


ACCUMULO-1481 creates root table and copies files to the directory on master start-up


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e04fd190
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e04fd190
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e04fd190

Branch: refs/heads/master
Commit: e04fd1902b23b8bca8e1c30795f45d9c596f8623
Parents: 8683295
Author: Eric Newton <eric.newton@gmail.com>
Authored: Mon Oct 28 17:15:23 2013 -0400
Committer: Eric Newton <eric.newton@gmail.com>
Committed: Mon Oct 28 17:15:23 2013 -0400

----------------------------------------------------------------------
 .../org/apache/accumulo/core/conf/Property.java |  2 --
 .../core/metadata/MetadataLocationObtainer.java |  2 +-
 .../accumulo/core/metadata/RootTable.java       |  1 +
 .../apache/accumulo/server/master/Master.java   | 25 ++++++++++++++++++--
 .../server/master/recovery/HadoopLogCloser.java |  2 +-
 .../master/state/MetaDataTableScanner.java      |  6 +++--
 .../accumulo/server/tabletserver/Tablet.java    |  9 ++++---
 .../server/tabletserver/TabletServer.java       |  4 ++--
 .../tabletserver/log/SortedLogRecovery.java     |  9 ++++++-
 .../apache/accumulo/server/util/Initialize.java | 15 ++++++++----
 .../accumulo/server/util/MetadataTableUtil.java | 15 ++++--------
 11 files changed, 61 insertions(+), 29 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/e04fd190/core/src/main/java/org/apache/accumulo/core/conf/Property.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 23bae10..134a037 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -594,12 +594,10 @@ public enum Property {
 
   public static Map<String,String> getCompactionStrategyOptions(AccumuloConfiguration
tableConf) {
     Map<String,String> longNames = tableConf.getAllPropertiesWithPrefix(Property.TABLE_COMPACTION_STRATEGY_PREFIX);
-    log.info("longNames " + longNames);
     Map<String,String> result = new HashMap<String, String>();
     for (Entry<String,String> entry : longNames.entrySet()) {
       result.put(entry.getKey().substring(Property.TABLE_COMPACTION_STRATEGY_PREFIX.getKey().length()),
entry.getValue());
     }
-    log.info("result " + result);
     return result;
   }
 }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/e04fd190/core/src/main/java/org/apache/accumulo/core/metadata/MetadataLocationObtainer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/MetadataLocationObtainer.java
b/core/src/main/java/org/apache/accumulo/core/metadata/MetadataLocationObtainer.java
index 7d6312d..0fb1a8e 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/MetadataLocationObtainer.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/MetadataLocationObtainer.java
@@ -113,7 +113,7 @@ public class MetadataLocationObtainer implements TabletLocationObtainer
{
       if (opTimer != null)
         opTimer.stop("Got " + results.size() + " results  from " + src.tablet_extent + "
in %DURATION%");
       
-      // System.out.println("results "+results.keySet());
+      //if (log.isTraceEnabled()) log.trace("results "+results);
       
       return MetadataLocationObtainer.getMetadataLocationEntries(results);
       

http://git-wip-us.apache.org/repos/asf/accumulo/blob/e04fd190/core/src/main/java/org/apache/accumulo/core/metadata/RootTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/RootTable.java b/core/src/main/java/org/apache/accumulo/core/metadata/RootTable.java
index 402c1cc..a47f183 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/RootTable.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/RootTable.java
@@ -42,5 +42,6 @@ public class RootTable {
   public static final String ZROOT_TABLET_WALOGS = ZROOT_TABLET + "/walogs";
   
   public static final KeyExtent EXTENT = new KeyExtent(new Text(ID), null, null);
+  public static final KeyExtent OLD_EXTENT = new KeyExtent(new Text(MetadataTable.ID), KeyExtent.getMetadataEntry(new
Text(MetadataTable.ID), null), null);
   
 }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/e04fd190/server/src/main/java/org/apache/accumulo/server/master/Master.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/master/Master.java b/server/src/main/java/org/apache/accumulo/server/master/Master.java
index 9cbfb51..2c19b6f 100644
--- a/server/src/main/java/org/apache/accumulo/server/master/Master.java
+++ b/server/src/main/java/org/apache/accumulo/server/master/Master.java
@@ -78,6 +78,7 @@ import org.apache.accumulo.core.security.Credentials;
 import org.apache.accumulo.core.security.SecurityUtil;
 import org.apache.accumulo.core.security.thrift.TCredentials;
 import org.apache.accumulo.core.util.ByteBufferUtil;
+import org.apache.accumulo.core.util.CachedConfiguration;
 import org.apache.accumulo.core.util.Daemon;
 import org.apache.accumulo.core.util.UtilWaitThread;
 import org.apache.accumulo.core.zookeeper.ZooUtil;
@@ -95,6 +96,7 @@ import org.apache.accumulo.server.ServerOpts;
 import org.apache.accumulo.server.client.HdfsZooInstance;
 import org.apache.accumulo.server.conf.ServerConfiguration;
 import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManager.FileType;
 import org.apache.accumulo.server.fs.VolumeManagerImpl;
 import org.apache.accumulo.server.master.LiveTServerSet.TServerConnection;
 import org.apache.accumulo.server.master.balancer.DefaultLoadBalancer;
@@ -135,6 +137,7 @@ import org.apache.accumulo.server.security.SecurityOperation;
 import org.apache.accumulo.server.security.SystemCredentials;
 import org.apache.accumulo.server.util.DefaultMap;
 import org.apache.accumulo.server.util.Halt;
+import org.apache.accumulo.server.util.Initialize;
 import org.apache.accumulo.server.util.MetadataTableUtil;
 import org.apache.accumulo.server.util.SystemPropUtil;
 import org.apache.accumulo.server.util.TServerUtils;
@@ -146,6 +149,9 @@ import org.apache.accumulo.server.zookeeper.ZooLock;
 import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 import org.apache.accumulo.trace.instrument.thrift.TraceWrap;
 import org.apache.accumulo.trace.thrift.TInfo;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.DataInputBuffer;
 import org.apache.hadoop.io.DataOutputBuffer;
 import org.apache.hadoop.io.Text;
@@ -265,9 +271,13 @@ public class Master implements LiveTServerSet.Listener, TableObserver,
CurrentSt
         
         IZooReaderWriter zoo = ZooReaderWriter.getInstance();
         
+        if (!Tables.exists(instance, RootTable.ID)) {
+          TableManager.prepareNewTableState(instance.getInstanceID(), RootTable.ID, RootTable.NAME,
TableState.ONLINE, NodeExistsPolicy.FAIL);
+          Initialize.initMetadataConfig(RootTable.ID);
+        }
+        
         zoo.recursiveDelete(ZooUtil.getRoot(instance) + "/loggers", NodeMissingPolicy.SKIP);
         zoo.recursiveDelete(ZooUtil.getRoot(instance) + "/dead/loggers", NodeMissingPolicy.SKIP);
-        
         zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZRECOVERY, new byte[]
{'0'}, NodeExistsPolicy.SKIP);
         
         for (String id : Tables.getIdToNameMap(instance).keySet()) {
@@ -293,7 +303,18 @@ public class Master implements LiveTServerSet.Listener, TableObserver,
CurrentSt
           @Override
           public void run() {
             try {
-              MetadataTableUtil.convertRootTabletToRootTable(instance, SystemCredentials.get());
+              Path oldDir = fs.getFullPath(FileType.TABLE, ServerConstants.getDefaultBaseDir()
+ "/tables/!0/root_tablet");
+              for (FileStatus file : fs.listStatus(oldDir)) {
+                if (file.isFile()) {
+                  Path newFile = new Path(ServerConstants.getRootTabletDir(), file.getPath().getName());
+                  FileUtil.copy(
+                      fs.getFileSystemByPath(file.getPath()), file.getPath(),
+                      fs.getFileSystemByPath(newFile), newFile,
+                      false,
+                      true,
+                      CachedConfiguration.getInstance());
+                }
+              }
               MetadataTableUtil.moveMetaDeleteMarkers(instance, SystemCredentials.get());
               Accumulo.updateAccumuloVersion(fs);
               

http://git-wip-us.apache.org/repos/asf/accumulo/blob/e04fd190/server/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java
b/server/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java
index 3e7d406..3eda844 100644
--- a/server/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java
+++ b/server/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java
@@ -46,7 +46,7 @@ public class HadoopLogCloser implements LogCloser {
       } catch (FileNotFoundException ex) {
         throw ex;
       } catch (Exception ex) {
-        log.warn("Error recovery lease on " + source.toString(), ex);
+        log.warn("Error recovering lease on " + source.toString(), ex);
         ns.append(source).close();
         log.info("Recovered lease on " + source.toString() + " using append");
       }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/e04fd190/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
b/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
index cec0bcf..1932958 100644
--- a/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
+++ b/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
@@ -37,6 +37,7 @@ import org.apache.accumulo.core.data.Range;
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.iterators.user.WholeRowIterator;
 import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ChoppedColumnFamily;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily;
@@ -49,7 +50,7 @@ import org.apache.log4j.Logger;
 public class MetaDataTableScanner implements Iterator<TabletLocationState> {
   private static final Logger log = Logger.getLogger(MetaDataTableScanner.class);
   
-  BatchScanner mdScanner;
+  BatchScanner mdScanner = null;
   Iterator<Entry<Key,Value>> iter;
   
   public MetaDataTableScanner(Instance instance, Credentials credentials, Range range, CurrentState
state) {
@@ -65,7 +66,8 @@ public class MetaDataTableScanner implements Iterator<TabletLocationState>
{
       mdScanner.setRanges(Collections.singletonList(range));
       iter = mdScanner.iterator();
     } catch (Exception ex) {
-      mdScanner.close();
+      if (mdScanner != null)
+        mdScanner.close();
       throw new RuntimeException(ex);
     }
   }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/e04fd190/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java b/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
index c734ff7..72c78c3 100644
--- a/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
+++ b/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
@@ -1271,7 +1271,6 @@ public class Tablet {
     this.lastCompactID = initCompactID;
     
     if (extent.isRootTablet()) {
-      
       long rtime = Long.MIN_VALUE;
       for (FileRef ref : datafiles.keySet()) {
         Path path = ref.path();
@@ -1295,6 +1294,10 @@ public class Tablet {
         }
       }
     }
+    if (time == null && datafiles.isEmpty() && extent.equals(RootTable.OLD_EXTENT))
{
+      // recovery... old root tablet has no data, so time doesn't matter:
+      time = TabletTime.LOGICAL_TIME_ID + "" + Long.MIN_VALUE;
+    }
     
     this.tabletServer = tabletServer;
     this.logId = tabletServer.createLogId(extent);
@@ -1430,7 +1433,7 @@ public class Tablet {
       removeOldTemporaryFiles();
     }
     
-    log.log(TLevel.TABLET_HIST, extent + " opened ");
+    log.log(TLevel.TABLET_HIST, extent + " opened");
   }
   
   private void removeOldTemporaryFiles() {
@@ -3176,7 +3179,7 @@ public class Tablet {
       droppedFiles.addAll(inputFiles);
       if (plan != null)
         droppedFiles.addAll(plan.deleteFiles);
-      propogateDeletes = !(droppedFiles.equals(allFiles.keySet()));
+      propogateDeletes = droppedFiles.equals(allFiles.keySet());
       log.debug("Major compaction plan: " + plan + " propogate deletes : " + propogateDeletes);
       filesToCompact = new HashMap<FileRef,DataFileValue>(allFiles);
       filesToCompact.keySet().retainAll(inputFiles);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/e04fd190/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
b/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
index 92aabf3..c1df5d3 100644
--- a/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
+++ b/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
@@ -2987,7 +2987,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
     long now = RelativeTime.currentTimeMillis();
     List<String> logSet = new ArrayList<String>();
     for (DfsLogger log : logs)
-      logSet.add(log.toString());
+      logSet.add(log.getFileName());
     MetadataTableUtil.LogEntry entry = new MetadataTableUtil.LogEntry();
     entry.extent = extent;
     entry.tabletId = id;
@@ -3376,7 +3376,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
       throw new AccumuloException("Metadata entry does not have directory (" + metadataEntry
+ ")");
     }
 
-    if (time == null) {
+    if (time == null && !extent.equals(RootTable.OLD_EXTENT)) {
       throw new AccumuloException("Metadata entry does not have time (" + metadataEntry +
")");
     }
 

http://git-wip-us.apache.org/repos/asf/accumulo/blob/e04fd190/server/src/main/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecovery.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecovery.java
b/server/src/main/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecovery.java
index 840fad0..2f1a875 100644
--- a/server/src/main/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecovery.java
+++ b/server/src/main/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecovery.java
@@ -29,10 +29,13 @@ import java.util.Set;
 
 import org.apache.accumulo.core.data.KeyExtent;
 import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
 import org.apache.accumulo.server.fs.VolumeManager;
 import org.apache.accumulo.server.logger.LogFileKey;
 import org.apache.accumulo.server.logger.LogFileValue;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
 import org.apache.log4j.Logger;
 
 /**
@@ -150,6 +153,10 @@ public class SortedLogRecovery {
         throw new RuntimeException("COMPACTION_FINISH (without preceding COMPACTION_START)
is not followed by a successful minor compaction.");
       lastStartToFinish.update(key.tserverSession);
     }
+    KeyExtent alternative = extent;
+    if (extent.isRootTablet()) {
+      alternative = RootTable.OLD_EXTENT;
+    }
     
     LogFileKey defineKey = null;
     
@@ -159,7 +166,7 @@ public class SortedLogRecovery {
       // LogReader.printEntry(entry);
       if (key.event != DEFINE_TABLET)
         break;
-      if (key.tablet.equals(extent)) {
+      if (key.tablet.equals(extent) || key.tablet.equals(alternative)) {
         if (tid != key.tid) {
           tid = key.tid;
           defineKey = key;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/e04fd190/server/src/main/java/org/apache/accumulo/server/util/Initialize.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/util/Initialize.java b/server/src/main/java/org/apache/accumulo/server/util/Initialize.java
index f7ff87a..3f6ef19 100644
--- a/server/src/main/java/org/apache/accumulo/server/util/Initialize.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/Initialize.java
@@ -237,7 +237,7 @@ public class Initialize {
     for (Path mtd : metadataTableDirs) {
       try {
         fstat = fs.getFileStatus(mtd);
-        if (!fstat.isDir()) {
+        if (!fstat.isDirectory()) {
           log.fatal("location " + mtd.toString() + " exists but is not a directory");
           return;
         }
@@ -252,7 +252,7 @@ public class Initialize {
     // create root table and tablet
     try {
       fstat = fs.getFileStatus(rootTablet);
-      if (!fstat.isDir()) {
+      if (!fstat.isDirectory()) {
         log.fatal("location " + rootTablet.toString() + " exists but is not a directory");
         return;
       }
@@ -313,14 +313,14 @@ public class Initialize {
       Path dir = new Path(s);
       try {
         fstat = fs.getFileStatus(dir);
-        if (!fstat.isDir()) {
+        if (!fstat.isDirectory()) {
           log.fatal("location " + dir.toString() + " exists but is not a directory");
           return;
         }
       } catch (FileNotFoundException fnfe) {
         try {
           fstat = fs.getFileStatus(dir);
-          if (!fstat.isDir()) {
+          if (!fstat.isDirectory()) {
             log.fatal("location " + dir.toString() + " exists but is not a directory");
             return;
           }
@@ -432,7 +432,7 @@ public class Initialize {
         opts.rootpass);
   }
   
-  protected static void initMetadataConfig() throws IOException {
+  public static void initMetadataConfig(String tableId) throws IOException {
     try {
       Configuration conf = CachedConfiguration.getInstance();
       int max = conf.getInt("dfs.replication.max", 512);
@@ -454,6 +454,11 @@ public class Initialize {
     }
   }
   
+  protected static void initMetadataConfig() throws IOException {
+    initMetadataConfig(RootTable.ID);
+    initMetadataConfig(MetadataTable.ID);
+  }
+  
   private static void setMetadataReplication(int replication, String reason) throws IOException
{
     String rep = getConsoleReader().readLine(
         "Your HDFS replication " + reason + " is not compatible with our default " + MetadataTable.NAME
+ " replication of 5. What do you want to set your "

http://git-wip-us.apache.org/repos/asf/accumulo/blob/e04fd190/server/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java b/server/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index f5ed91b..b52ec9a 100644
--- a/server/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@ -810,7 +810,7 @@ public class MetadataTableUtil {
     if (extent.isRootTablet()) {
       getRootLogEntries(result);
       Path rootDir = new Path(ServerConstants.getRootTabletDir());
-      rootDir = rootDir.makeQualified(fs.getDefaultVolume());
+      rootDir = fs.getDefaultVolume().makeQualified(rootDir);
       FileStatus[] files = fs.listStatus(rootDir);
       for (FileStatus fileStatus : files) {
         if (fileStatus.getPath().toString().endsWith("_tmp")) {
@@ -892,6 +892,8 @@ public class MetadataTableUtil {
         LogEntry e = new LogEntry();
         try {
           e.fromBytes(zoo.getData(root + "/" + child, null));
+          // upgrade from !0;!0<< -> !!R<<
+          e.extent = RootTable.EXTENT;
           result.add(e);
         } catch (KeeperException.NoNodeException ex) {
           continue;
@@ -1275,11 +1277,11 @@ public class MetadataTableUtil {
         log.info("Moving " + filename + " marker in " + RootTable.NAME);
         Mutation m = new Mutation(MetadataSchema.DeletesSection.getRowPrefix() + filename);
         m.put(EMPTY_BYTES, EMPTY_BYTES, EMPTY_BYTES);
-        update(creds, m, null);
+        update(creds, m, RootTable.EXTENT);
         // remove the old entry
         m = new Mutation(entry.getKey().getRow());
         m.putDelete(EMPTY_BYTES, EMPTY_BYTES);
-        update(creds, m, null);
+        update(creds, m,  RootTable.OLD_EXTENT);
       } else {
         break;
       }
@@ -1315,11 +1317,4 @@ public class MetadataTableUtil {
     return tabletEntries;
   }
 
-  public static void convertRootTabletToRootTable(Instance instance, SystemCredentials systemCredentials)
throws KeeperException, InterruptedException {
-    ZooReaderWriter zoo = ZooReaderWriter.getInstance();
-    if (zoo.exists(ZooUtil.getRoot(instance)+"/tables/" + RootTable.ID))
-      return;
-    TableManager.prepareNewTableState(instance.getInstanceID(), RootTable.ID, RootTable.NAME,
TableState.ONLINE, NodeExistsPolicy.FAIL);
-  }
-  
 }


Mime
View raw message