accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ctubb...@apache.org
Subject accumulo git commit: ACCUMULO-4087 Use same volume for tablet tmp files
Date Fri, 21 Apr 2017 19:12:37 GMT
Repository: accumulo
Updated Branches:
  refs/heads/master 92c45a896 -> e50102229


ACCUMULO-4087 Use same volume for tablet tmp files

FileUtil does not need to choose a new volume when it needs to create a
temporary file during midpoint computation and splitting. It can instead
use the tablet's already-chosen volume/directory.


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e5010222
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e5010222
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e5010222

Branch: refs/heads/master
Commit: e50102229a7aab898e2e0191f1d38a6dfe8f8847
Parents: 92c45a8
Author: Christopher Tubbs <ctubbsii@apache.org>
Authored: Fri Apr 21 15:11:20 2017 -0400
Committer: Christopher Tubbs <ctubbsii@apache.org>
Committed: Fri Apr 21 15:11:20 2017 -0400

----------------------------------------------------------------------
 .../apache/accumulo/server/util/FileUtil.java   | 31 ++++++++------------
 .../apache/accumulo/tserver/tablet/Tablet.java  |  6 ++--
 2 files changed, 15 insertions(+), 22 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/e5010222/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
index 4e0528e..36c8f29 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
@@ -24,7 +24,6 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
-import java.util.Optional;
 import java.util.Random;
 import java.util.Set;
 import java.util.SortedMap;
@@ -47,9 +46,7 @@ import org.apache.accumulo.core.iterators.system.MultiIterator;
 import org.apache.accumulo.core.util.CachedConfiguration;
 import org.apache.accumulo.core.util.LocalityGroupUtil;
 import org.apache.accumulo.core.volume.Volume;
-import org.apache.accumulo.server.ServerConstants;
 import org.apache.accumulo.server.fs.FileRef;
-import org.apache.accumulo.server.fs.VolumeChooserEnvironment;
 import org.apache.accumulo.server.fs.VolumeManager;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
@@ -81,14 +78,10 @@ public class FileUtil {
 
   private static final Logger log = LoggerFactory.getLogger(FileUtil.class);
 
-  private static Path createTmpDir(AccumuloConfiguration acuConf, VolumeManager fs) throws
IOException {
-    VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(Optional.empty());
-    String accumuloDir = fs.choose(chooserEnv, ServerConstants.getBaseUris());
-
+  private static Path createTmpDir(AccumuloConfiguration acuConf, VolumeManager fs, String
tabletDirectory) throws IOException {
     Path result = null;
     while (result == null) {
-      result = new Path(accumuloDir + Path.SEPARATOR + "tmp/idxReduce_" + String.format("%09d",
new Random().nextInt(Integer.MAX_VALUE)));
-
+      result = new Path(tabletDirectory + Path.SEPARATOR + "tmp/idxReduce_" + String.format("%09d",
new Random().nextInt(Integer.MAX_VALUE)));
       try {
         fs.getFileStatus(result);
         result = null;
@@ -190,13 +183,13 @@ public class FileUtil {
     return reduceFiles(acuConf, conf, fs, prevEndRow, endRow, outFiles, maxFiles, tmpDir,
pass + 1);
   }
 
-  public static SortedMap<Double,Key> findMidPoint(VolumeManager fs, AccumuloConfiguration
acuConf, Text prevEndRow, Text endRow, Collection<String> mapFiles,
-      double minSplit) throws IOException {
-    return findMidPoint(fs, acuConf, prevEndRow, endRow, mapFiles, minSplit, true);
+  public static SortedMap<Double,Key> findMidPoint(VolumeManager fs, String tabletDir,
AccumuloConfiguration acuConf, Text prevEndRow, Text endRow,
+      Collection<String> mapFiles, double minSplit) throws IOException {
+    return findMidPoint(fs, tabletDir, acuConf, prevEndRow, endRow, mapFiles, minSplit, true);
   }
 
-  public static double estimatePercentageLTE(VolumeManager fs, AccumuloConfiguration acuconf,
Text prevEndRow, Text endRow, Collection<String> mapFiles,
-      Text splitRow) throws IOException {
+  public static double estimatePercentageLTE(VolumeManager fs, String tabletDir, AccumuloConfiguration
acuconf, Text prevEndRow, Text endRow,
+      Collection<String> mapFiles, Text splitRow) throws IOException {
 
     Configuration conf = CachedConfiguration.getInstance();
 
@@ -207,7 +200,7 @@ public class FileUtil {
 
     try {
       if (mapFiles.size() > maxToOpen) {
-        tmpDir = createTmpDir(acuconf, fs);
+        tmpDir = createTmpDir(acuconf, fs, tabletDir);
 
         log.debug("Too many indexes (" + mapFiles.size() + ") to open at once for " + endRow
+ " " + prevEndRow + ", reducing in tmpDir = " + tmpDir);
 
@@ -267,8 +260,8 @@ public class FileUtil {
    *          ISSUES : This method used the index files to find the mid point. If the map
files have different index intervals this method will not return an
    *          accurate mid point. Also, it would be tricky to use this method in conjunction
with an in memory map because the indexing interval is unknown.
    */
-  public static SortedMap<Double,Key> findMidPoint(VolumeManager fs, AccumuloConfiguration
acuConf, Text prevEndRow, Text endRow, Collection<String> mapFiles,
-      double minSplit, boolean useIndex) throws IOException {
+  public static SortedMap<Double,Key> findMidPoint(VolumeManager fs, String tabletDirectory,
AccumuloConfiguration acuConf, Text prevEndRow, Text endRow,
+      Collection<String> mapFiles, double minSplit, boolean useIndex) throws IOException
{
     Configuration conf = CachedConfiguration.getInstance();
 
     Collection<String> origMapFiles = mapFiles;
@@ -282,7 +275,7 @@ public class FileUtil {
       if (mapFiles.size() > maxToOpen) {
         if (!useIndex)
           throw new IOException("Cannot find mid point using data files, too many " + mapFiles.size());
-        tmpDir = createTmpDir(acuConf, fs);
+        tmpDir = createTmpDir(acuConf, fs, tabletDirectory);
 
         log.debug("Too many indexes (" + mapFiles.size() + ") to open at once for " + endRow
+ " " + prevEndRow + ", reducing in tmpDir = " + tmpDir);
 
@@ -307,7 +300,7 @@ public class FileUtil {
           log.warn("Failed to find mid point using indexes, falling back to data files which
is slower. No entries between " + prevEndRow + " and " + endRow
               + " for " + mapFiles);
           // need to pass original map files, not possibly reduced indexes
-          return findMidPoint(fs, acuConf, prevEndRow, endRow, origMapFiles, minSplit, false);
+          return findMidPoint(fs, tabletDirectory, acuConf, prevEndRow, endRow, origMapFiles,
minSplit, false);
         }
         throw new IOException("Failed to find mid point, no entries between " + prevEndRow
+ " and " + endRow + " for " + mapFiles);
       }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/e5010222/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index da3888e..ce064db 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -1487,8 +1487,8 @@ public class Tablet implements TabletCommitter {
 
     try {
       // we should make .25 below configurable
-      keys = FileUtil.findMidPoint(getTabletServer().getFileSystem(), getTabletServer().getConfiguration(),
extent.getPrevEndRow(), extent.getEndRow(),
-          FileUtil.toPathStrings(files), .25);
+      keys = FileUtil.findMidPoint(getTabletServer().getFileSystem(), tabletDirectory, getTabletServer().getConfiguration(),
extent.getPrevEndRow(),
+          extent.getEndRow(), FileUtil.toPathStrings(files), .25);
     } catch (IOException e) {
       log.error("Failed to find midpoint " + e.getMessage());
       return null;
@@ -2105,7 +2105,7 @@ public class Tablet implements TabletCommitter {
         splitPoint = findSplitRow(getDatafileManager().getFiles());
       else {
         Text tsp = new Text(sp);
-        splitPoint = new SplitRowSpec(FileUtil.estimatePercentageLTE(getTabletServer().getFileSystem(),
getTabletServer().getConfiguration(),
+        splitPoint = new SplitRowSpec(FileUtil.estimatePercentageLTE(getTabletServer().getFileSystem(),
tabletDirectory, getTabletServer().getConfiguration(),
             extent.getPrevEndRow(), extent.getEndRow(), FileUtil.toPathStrings(getDatafileManager().getFiles()),
tsp), tsp);
       }
 


Mime
View raw message