accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject [1/2] accumulo git commit: ACCUMULO-3339 found some more metadata clean-up to move into TabletData
Date Tue, 07 Jul 2015 16:33:43 GMT
Repository: accumulo
Updated Branches:
  refs/heads/master e0ef0ca42 -> dd99cbff2


ACCUMULO-3339 found some more metadata clean-up to move into TabletData


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/0721a6b6
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/0721a6b6
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/0721a6b6

Branch: refs/heads/master
Commit: 0721a6b6183dd0c268ed7092b0b588f49596de0d
Parents: 3e5acd5
Author: Eric C. Newton <eric.newton@gmail.com>
Authored: Mon Jul 6 15:24:29 2015 -0400
Committer: Eric C. Newton <eric.newton@gmail.com>
Committed: Mon Jul 6 15:24:29 2015 -0400

----------------------------------------------------------------------
 .../apache/accumulo/tserver/TabletServer.java   |  5 +-
 .../apache/accumulo/tserver/tablet/Tablet.java  | 85 ++++++--------------
 .../accumulo/tserver/tablet/TabletData.java     | 45 ++++++++---
 3 files changed, 58 insertions(+), 77 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/0721a6b6/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index a8be243..5d3e0f0 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -159,7 +159,6 @@ import org.apache.accumulo.server.fs.FileRef;
 import org.apache.accumulo.server.fs.VolumeManager;
 import org.apache.accumulo.server.fs.VolumeManager.FileType;
 import org.apache.accumulo.server.fs.VolumeManagerImpl;
-import org.apache.accumulo.server.fs.VolumeUtil;
 import org.apache.accumulo.server.log.SortedLogState;
 import org.apache.accumulo.server.log.WalStateManager;
 import org.apache.accumulo.server.log.WalStateManager.WalMarkerException;
@@ -2131,12 +2130,10 @@ public class TabletServer extends AccumuloServerContext implements
Runnable {
         TabletResourceManager trm = resourceManager.createTabletResourceManager(extent, getTableConfiguration(extent));
         TabletData data;
         if (extent.isRootTablet()) {
-          data = new TabletData(fs, ZooReaderWriter.getInstance());
+          data = new TabletData(fs, ZooReaderWriter.getInstance(), getTableConfiguration(extent));
         } else {
           data = new TabletData(extent, fs, tabletsKeyValues.entrySet().iterator());
         }
-        // this opens the tablet file and fills in the endKey in the extent
-        data.setDirectory(VolumeUtil.switchRootTabletVolume(extent, data.getDirectory()));
 
         tablet = new Tablet(TabletServer.this, extent, trm, data);
         // If a minor compaction starts after a tablet opens, this indicates a log recovery
occurred. This recovered data must be minor compacted.

http://git-wip-us.apache.org/repos/asf/accumulo/blob/0721a6b6/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 307044f..8fcf035 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -173,7 +173,7 @@ public class Tablet implements TabletCommitter {
   private final Object timeLock = new Object();
   private long persistedTime;
 
-  private TServerInstance lastLocation;
+  private TServerInstance lastLocation = null;
   private volatile boolean tableDirChecked = false;
 
   private final AtomicLong dataSourceDeletions = new AtomicLong(0);
@@ -309,6 +309,17 @@ public class Tablet implements TabletCommitter {
 
   public Tablet(final TabletServer tabletServer, final KeyExtent extent, final TabletResourceManager
trm, TabletData data) throws IOException {
 
+    this.tabletServer = tabletServer;
+    this.extent = extent;
+    this.tabletResources = trm;
+    this.lastLocation = data.getLastLocation();
+    this.lastFlushID = data.getFlushID();
+    this.lastCompactID = data.getCompactID();
+    this.splitCreationTime = data.getSplitTime();
+    this.tabletTime = TabletTime.getInstance(data.getTime());
+    this.persistedTime = tabletTime.getTime();
+    this.logId = tabletServer.createLogId(extent);
+
     TableConfiguration tblConf = tabletServer.getTableConfiguration(extent);
     if (null == tblConf) {
       Tables.clearCache(tabletServer.getInstance());
@@ -318,71 +329,28 @@ public class Tablet implements TabletCommitter {
 
     this.tableConfiguration = tblConf;
 
-    TabletFiles tabletPaths = VolumeUtil
-        .updateTabletVolumes(tabletServer, tabletServer.getLock(), tabletServer.getFileSystem(),
extent,
-            new TabletFiles(data.getDirectory(), data.getLogEntris(), data.getDataFiles()),
-            ReplicationConfigurationUtil.isEnabled(extent, this.tableConfiguration));
+    // translate any volume changes
+    VolumeManager fs = tabletServer.getFileSystem();
+    boolean replicationEnabled = ReplicationConfigurationUtil.isEnabled(extent, this.tableConfiguration);
+    TabletFiles tabletPaths = new TabletFiles(data.getDirectory(), data.getLogEntris(), data.getDataFiles());
+    tabletPaths = VolumeUtil.updateTabletVolumes(tabletServer, tabletServer.getLock(), fs,
extent, tabletPaths, replicationEnabled);
 
+    // deal with relative path for the directory
     Path locationPath;
-
     if (tabletPaths.dir.contains(":")) {
-      locationPath = new Path(tabletPaths.dir.toString());
+      locationPath = new Path(tabletPaths.dir);
     } else {
-      locationPath = tabletServer.getFileSystem().getFullPath(FileType.TABLE, extent.getTableId().toString()
+ tabletPaths.dir.toString());
+      locationPath = tabletServer.getFileSystem().getFullPath(FileType.TABLE, extent.getTableId().toString()
+ tabletPaths.dir);
     }
-
-    final List<LogEntry> logEntries = tabletPaths.logEntries;
-    final SortedMap<FileRef,DataFileValue> datafiles = tabletPaths.datafiles;
-
     this.location = locationPath;
-    this.lastLocation = data.getLastLocation();
     this.tabletDirectory = tabletPaths.dir;
-
-    this.extent = extent;
-    this.tabletResources = trm;
-
-    this.lastFlushID = data.getFlushID();
-    this.lastCompactID = data.getCompactID();
-    this.splitCreationTime = data.getSplitTime();
-    String time = data.getTime();
-
-    if (extent.isRootTablet()) {
-      long rtime = Long.MIN_VALUE;
-      for (FileRef ref : datafiles.keySet()) {
-        Path path = ref.path();
-        FileSystem ns = tabletServer.getFileSystem().getVolumeByPath(path).getFileSystem();
-        FileSKVIterator reader = FileOperations.getInstance().openReader(path.toString(),
true, ns, ns.getConf(), tabletServer.getTableConfiguration(extent));
-        long maxTime = -1;
-        try {
-
-          while (reader.hasTop()) {
-            maxTime = Math.max(maxTime, reader.getTopKey().getTimestamp());
-            reader.next();
-          }
-
-        } finally {
-          reader.close();
-        }
-
-        if (maxTime > rtime) {
-          time = TabletTime.LOGICAL_TIME_ID + "" + maxTime;
-          rtime = maxTime;
-        }
-      }
-    }
-    if (time == null && datafiles.isEmpty() && extent.equals(RootTable.OLD_EXTENT))
{
-      // recovery... old root tablet has no data, so time doesn't matter:
-      time = TabletTime.LOGICAL_TIME_ID + "" + Long.MIN_VALUE;
+    for (Entry<Long,List<FileRef>> entry : data.getBulkImported().entrySet())
{
+      this.bulkImported.put(entry.getKey(), new CopyOnWriteArrayList<FileRef>(entry.getValue()));
     }
-
-    this.tabletServer = tabletServer;
-    this.logId = tabletServer.createLogId(extent);
-
     setupDefaultSecurityLabels(extent);
 
-    tabletMemory = new TabletMemory(this);
-    tabletTime = TabletTime.getInstance(time);
-    persistedTime = tabletTime.getTime();
+    final List<LogEntry> logEntries = tabletPaths.logEntries;
+    final SortedMap<FileRef,DataFileValue> datafiles = tabletPaths.datafiles;
 
     tableConfiguration.addObserver(configObserver = new ConfigurationObserver() {
 
@@ -425,13 +393,10 @@ public class Tablet implements TabletCommitter {
     });
 
     tableConfiguration.getNamespaceConfiguration().addObserver(configObserver);
+    tabletMemory = new TabletMemory(this);
 
     // Force a load of any per-table properties
     configObserver.propertiesChanged();
-    for (Entry<Long,List<FileRef>> entry : data.getBulkImported().entrySet())
{
-      this.bulkImported.put(entry.getKey(), new CopyOnWriteArrayList<FileRef>(entry.getValue()));
-    }
-
     if (!logEntries.isEmpty()) {
       log.info("Starting Write-Ahead Log recovery for " + this.extent);
       final AtomicLong entriesUsedOnTablet = new AtomicLong(0);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/0721a6b6/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletData.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletData.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletData.java
index cb9015e..bfbf33f 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletData.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletData.java
@@ -33,9 +33,12 @@ import java.util.Map.Entry;
 import java.util.SortedMap;
 import java.util.TreeMap;
 
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.data.impl.KeyExtent;
+import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.core.file.FileSKVIterator;
 import org.apache.accumulo.core.metadata.RootTable;
 import org.apache.accumulo.core.metadata.schema.DataFileValue;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.BulkFileColumnFamily;
@@ -48,9 +51,12 @@ import org.apache.accumulo.core.tabletserver.log.LogEntry;
 import org.apache.accumulo.fate.zookeeper.ZooReader;
 import org.apache.accumulo.server.fs.FileRef;
 import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeUtil;
 import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.tablets.TabletTime;
 import org.apache.accumulo.server.util.MetadataTableUtil;
 import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Text;
 import org.slf4j.Logger;
@@ -119,38 +125,55 @@ public class TabletData {
         }
       }
     }
+    if (time == null && dataFiles.isEmpty() && extent.equals(RootTable.OLD_EXTENT))
{
+      // recovery... old root tablet has no data, so time doesn't matter:
+      time = TabletTime.LOGICAL_TIME_ID + "" + Long.MIN_VALUE;
+    }
   }
 
   // Read basic root table metadata from zookeeper
-  public TabletData(VolumeManager fs, ZooReader rdr) throws IOException {
+  public TabletData(VolumeManager fs, ZooReader rdr, AccumuloConfiguration conf) throws IOException
{
     Path location = new Path(MetadataTableUtil.getRootTabletDir());
 
     // cleanReplacement() has special handling for deleting files
     FileStatus[] files = fs.listStatus(location);
     Collection<String> goodPaths = RootFiles.cleanupReplacement(fs, files, true);
+    long rtime = Long.MIN_VALUE;
     for (String good : goodPaths) {
       Path path = new Path(good);
       String filename = path.getName();
       FileRef ref = new FileRef(location.toString() + "/" + filename, path);
       DataFileValue dfv = new DataFileValue(0, 0);
       dataFiles.put(ref, dfv);
+
+      FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
+      FileSKVIterator reader = FileOperations.getInstance().openReader(path.toString(), true,
ns, ns.getConf(), conf);
+      long maxTime = -1;
+      try {
+        while (reader.hasTop()) {
+          maxTime = Math.max(maxTime, reader.getTopKey().getTimestamp());
+          reader.next();
+        }
+      } finally {
+        reader.close();
+      }
+      if (maxTime > rtime) {
+        time = TabletTime.LOGICAL_TIME_ID + "" + maxTime;
+        rtime = maxTime;
+      }
     }
+
     try {
       logEntris = MetadataTableUtil.getLogEntries(null, RootTable.EXTENT);
     } catch (Exception ex) {
       throw new RuntimeException("Unable to read tablet log entries", ex);
     }
-    directory = MetadataTableUtil.getRootTabletDir();
+    directory = VolumeUtil.switchRootTabletVolume(RootTable.EXTENT, MetadataTableUtil.getRootTabletDir());
   }
 
   // Data pulled from an existing tablet to make a split
-  public TabletData(String tabletDirectory,
-      SortedMap<FileRef,DataFileValue> highDatafileSizes,
-      String time,
-      long lastFlushID,
-      long lastCompactID,
-      TServerInstance lastLocation,
-      Map<Long,List<FileRef>> bulkIngestedFiles) {
+  public TabletData(String tabletDirectory, SortedMap<FileRef,DataFileValue> highDatafileSizes,
String time, long lastFlushID, long lastCompactID,
+      TServerInstance lastLocation, Map<Long,List<FileRef>> bulkIngestedFiles)
{
     this.directory = tabletDirectory;
     this.dataFiles = highDatafileSizes;
     this.time = time;
@@ -201,10 +224,6 @@ public class TabletData {
     return directory;
   }
 
-  public void setDirectory(String directory) {
-    this.directory = directory;
-  }
-
   public long getSplitTime() {
     return splitTime;
   }


Mime
View raw message