accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject accumulo git commit: ACCUMULO-3949 reduce the number of calls to FileSystem#getFileStatus
Date Wed, 29 Jul 2015 20:31:03 GMT
Repository: accumulo
Updated Branches:
  refs/heads/master 4999c421a -> 650322351


ACCUMULO-3949 reduce the number of calls to FileSystem#getFileStatus


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/65032235
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/65032235
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/65032235

Branch: refs/heads/master
Commit: 650322351bdee524aa8decf7ed8606d0d07d296b
Parents: 4999c42
Author: Eric Newton <eric.newton@gmail.com>
Authored: Wed Jul 29 16:30:57 2015 -0400
Committer: Eric Newton <eric.newton@gmail.com>
Committed: Wed Jul 29 16:30:57 2015 -0400

----------------------------------------------------------------------
 .../core/client/impl/TableOperationsImpl.java   | 12 +++++-----
 .../accumulo/core/file/BloomFilterLayer.java    | 10 +++++++++
 .../accumulo/core/file/FileSKVWriter.java       |  2 ++
 .../core/file/blockfile/BlockFileWriter.java    |  2 ++
 .../file/blockfile/impl/CachableBlockFile.java  | 12 ++++++----
 .../apache/accumulo/core/file/rfile/RFile.java  | 11 +++++++++-
 .../accumulo/tserver/tablet/Compactor.java      | 23 +++++++++-----------
 7 files changed, 49 insertions(+), 23 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/65032235/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java
b/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java
index 2a486fb..d65bcec 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java
@@ -17,9 +17,11 @@
 package org.apache.accumulo.core.client.impl;
 
 import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 import java.io.BufferedReader;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.nio.ByteBuffer;
@@ -117,7 +119,6 @@ import org.slf4j.LoggerFactory;
 
 import com.google.common.base.Joiner;
 import com.google.common.net.HostAndPort;
-import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
 
 public class TableOperationsImpl extends TableOperationsHelper {
 
@@ -1039,11 +1040,12 @@ public class TableOperationsImpl extends TableOperationsHelper {
       ret = fs.makeQualified(new Path(dir));
     }
 
-    if (!fs.exists(ret))
+    try {
+      if (!fs.getFileStatus(ret).isDirectory()) {
+        throw new AccumuloException(kind + " import " + type + " directory " + dir + " is
not a directory!");
+      }
+    } catch (FileNotFoundException fnf) {
       throw new AccumuloException(kind + " import " + type + " directory " + dir + " does
not exist!");
-
-    if (!fs.getFileStatus(ret).isDirectory()) {
-      throw new AccumuloException(kind + " import " + type + " directory " + dir + " is not
a directory!");
     }
 
     if (type.equals("failure")) {

http://git-wip-us.apache.org/repos/asf/accumulo/blob/65032235/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java b/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java
index 765aa0c..a5bea83 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java
@@ -94,6 +94,7 @@ public class BloomFilterLayer {
     private FileSKVWriter writer;
     private KeyFunctor transformer = null;
     private boolean closed = false;
+    private long length = -1;
 
     Writer(FileSKVWriter writer, AccumuloConfiguration acuconf) {
       this.writer = writer;
@@ -154,6 +155,7 @@ public class BloomFilterLayer {
       out.flush();
       out.close();
       writer.close();
+      length = writer.getLength();
       closed = true;
     }
 
@@ -177,6 +179,14 @@ public class BloomFilterLayer {
     public boolean supportsLocalityGroups() {
       return writer.supportsLocalityGroups();
     }
+
+    @Override
+    public long getLength() throws IOException {
+      if (closed) {
+        return length;
+      }
+      return writer.getLength();
+    }
   }
 
   static class BloomFilterLoader {

http://git-wip-us.apache.org/repos/asf/accumulo/blob/65032235/core/src/main/java/org/apache/accumulo/core/file/FileSKVWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/FileSKVWriter.java b/core/src/main/java/org/apache/accumulo/core/file/FileSKVWriter.java
index f4aa888..98a366f 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/FileSKVWriter.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/FileSKVWriter.java
@@ -36,4 +36,6 @@ public interface FileSKVWriter {
   DataOutputStream createMetaStore(String name) throws IOException;
 
   void close() throws IOException;
+
+  long getLength() throws IOException;
 }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/65032235/core/src/main/java/org/apache/accumulo/core/file/blockfile/BlockFileWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/blockfile/BlockFileWriter.java
b/core/src/main/java/org/apache/accumulo/core/file/blockfile/BlockFileWriter.java
index 570a8a5..e9d97c5 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/blockfile/BlockFileWriter.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/blockfile/BlockFileWriter.java
@@ -33,4 +33,6 @@ public interface BlockFileWriter {
   ABlockWriter prepareDataBlock() throws IOException;
 
   void close() throws IOException;
+
+  long getLength() throws IOException;
 }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/65032235/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
b/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
index d00a797..8a6f38b 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
@@ -55,7 +55,8 @@ public class CachableBlockFile {
   public static class Writer implements BlockFileWriter {
     private BCFile.Writer _bc;
     private BlockWrite _bw;
-    private FSDataOutputStream fsout = null;
+    private final FSDataOutputStream fsout;
+    private long length = 0;
 
     public Writer(FileSystem fs, Path fName, String compressAlgor, Configuration conf, AccumuloConfiguration
accumuloConfiguration) throws IOException {
       this.fsout = fs.create(fName);
@@ -89,10 +90,13 @@ public class CachableBlockFile {
       _bw.close();
       _bc.close();
 
-      if (this.fsout != null) {
-        this.fsout.close();
-      }
+      length = this.fsout.getPos();
+      this.fsout.close();
+    }
 
+    @Override
+    public long getLength() throws IOException {
+      return length;
     }
 
   }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/65032235/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
index 060d956..54b01b4 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
@@ -303,6 +303,7 @@ public class RFile {
     private boolean startedDefaultLocalityGroup = false;
 
     private HashSet<ByteSequence> previousColumnFamilies;
+    private long length = -1;
 
     public Writer(BlockFileWriter bfw, int blockSize) throws IOException {
       this(bfw, blockSize, (int) AccumuloConfiguration.getDefaultConfiguration().getMemoryInBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX));
@@ -340,8 +341,8 @@ public class RFile {
       }
 
       mba.close();
-
       fileWriter.close();
+      length = fileWriter.getLength();
 
       closed = true;
     }
@@ -465,6 +466,14 @@ public class RFile {
     public boolean supportsLocalityGroups() {
       return true;
     }
+
+    @Override
+    public long getLength() throws IOException {
+      if (!closed) {
+        return fileWriter.getLength();
+      }
+      return length;
+    }
   }
 
   private static class LocalityGroupReader extends LocalityGroup implements FileSKVIterator
{

http://git-wip-us.apache.org/repos/asf/accumulo/blob/65032235/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Compactor.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Compactor.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Compactor.java
index 0a8a9e3..c1b8105 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Compactor.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Compactor.java
@@ -219,19 +219,16 @@ public class Compactor implements Callable<CompactionStats> {
       mfw = null; // set this to null so we do not try to close it again in finally if the
close fails
       mfwTmp.close(); // if the close fails it will cause the compaction to fail
 
-      // Verify the file, since hadoop 0.20.2 sometimes lies about the success of close()
-      try {
-        FileSKVIterator openReader = fileFactory.openReader(outputFile.path().toString(),
false, ns, ns.getConf(), acuTableConf);
-        openReader.close();
-      } catch (IOException ex) {
-        log.error("Verification of successful compaction fails!!! " + extent + " " + outputFile,
ex);
-        throw ex;
-      }
-
-      log.debug(String.format("Compaction %s %,d read | %,d written | %,6d entries/sec |
%6.3f secs", extent, majCStats.getEntriesRead(),
-          majCStats.getEntriesWritten(), (int) (majCStats.getEntriesRead() / ((t2 - t1) /
1000.0)), (t2 - t1) / 1000.0));
-
-      majCStats.setFileSize(fileFactory.getFileSize(outputFile.path().toString(), ns, ns.getConf(),
acuTableConf));
+      log.debug(String.format("Compaction %s %,d read | %,d written | %,6d entries/sec |
%,6.3f secs | %,12d bytes | %9.3f byte/sec",
+          extent,
+          majCStats.getEntriesRead(),
+          majCStats.getEntriesWritten(),
+          (int) (majCStats.getEntriesRead() / ((t2 - t1) / 1000.0)),
+          (t2 - t1) / 1000.0,
+          mfwTmp.getLength(),
+          mfwTmp.getLength() / ((t2 - t1) / 1000.0)));
+
+      majCStats.setFileSize(mfwTmp.getLength());
       return majCStats;
     } catch (IOException e) {
       log.error("{}", e.getMessage(), e);


Mime
View raw message