Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id F3B8718B8A for ; Wed, 29 Jul 2015 20:31:03 +0000 (UTC) Received: (qmail 98350 invoked by uid 500); 29 Jul 2015 20:31:03 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 98313 invoked by uid 500); 29 Jul 2015 20:31:03 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 98304 invoked by uid 99); 29 Jul 2015 20:31:03 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 29 Jul 2015 20:31:03 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id A6EE7DFFEF; Wed, 29 Jul 2015 20:31:03 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: ecn@apache.org To: commits@accumulo.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: accumulo git commit: ACCUMULO-3949 reduce the number of calls to FileSystem#getFileStatus Date: Wed, 29 Jul 2015 20:31:03 +0000 (UTC) Repository: accumulo Updated Branches: refs/heads/master 4999c421a -> 650322351 ACCUMULO-3949 reduce the number of calls to FileSystem#getFileStatus Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/65032235 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/65032235 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/65032235 Branch: refs/heads/master Commit: 650322351bdee524aa8decf7ed8606d0d07d296b Parents: 4999c42 Author: Eric Newton Authored: Wed Jul 29 16:30:57 2015 -0400 Committer: Eric Newton Committed: Wed Jul 29 16:30:57 2015 -0400 ---------------------------------------------------------------------- .../core/client/impl/TableOperationsImpl.java | 12 +++++----- .../accumulo/core/file/BloomFilterLayer.java | 10 +++++++++ .../accumulo/core/file/FileSKVWriter.java | 2 ++ .../core/file/blockfile/BlockFileWriter.java | 2 ++ .../file/blockfile/impl/CachableBlockFile.java | 12 ++++++---- .../apache/accumulo/core/file/rfile/RFile.java | 11 +++++++++- .../accumulo/tserver/tablet/Compactor.java | 23 +++++++++----------- 7 files changed, 49 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/65032235/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java b/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java index 2a486fb..d65bcec 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java +++ b/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java @@ -17,9 +17,11 @@ package org.apache.accumulo.core.client.impl; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly; import static java.nio.charset.StandardCharsets.UTF_8; import java.io.BufferedReader; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStreamReader; import java.nio.ByteBuffer; @@ -117,7 +119,6 @@ import org.slf4j.LoggerFactory; import com.google.common.base.Joiner; import com.google.common.net.HostAndPort; -import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly; public class TableOperationsImpl extends TableOperationsHelper { @@ -1039,11 +1040,12 @@ public class TableOperationsImpl extends TableOperationsHelper { ret = fs.makeQualified(new Path(dir)); } - if (!fs.exists(ret)) + try { + if (!fs.getFileStatus(ret).isDirectory()) { + throw new AccumuloException(kind + " import " + type + " directory " + dir + " is not a directory!"); + } + } catch (FileNotFoundException fnf) { throw new AccumuloException(kind + " import " + type + " directory " + dir + " does not exist!"); - - if (!fs.getFileStatus(ret).isDirectory()) { - throw new AccumuloException(kind + " import " + type + " directory " + dir + " is not a directory!"); } if (type.equals("failure")) { http://git-wip-us.apache.org/repos/asf/accumulo/blob/65032235/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java b/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java index 765aa0c..a5bea83 100644 --- a/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java +++ b/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java @@ -94,6 +94,7 @@ public class BloomFilterLayer { private FileSKVWriter writer; private KeyFunctor transformer = null; private boolean closed = false; + private long length = -1; Writer(FileSKVWriter writer, AccumuloConfiguration acuconf) { this.writer = writer; @@ -154,6 +155,7 @@ public class BloomFilterLayer { out.flush(); out.close(); writer.close(); + length = writer.getLength(); closed = true; } @@ -177,6 +179,14 @@ public class BloomFilterLayer { public boolean supportsLocalityGroups() { return writer.supportsLocalityGroups(); } + + @Override + public long getLength() throws IOException { + if (closed) { + return length; + } + return writer.getLength(); + } } static class BloomFilterLoader { http://git-wip-us.apache.org/repos/asf/accumulo/blob/65032235/core/src/main/java/org/apache/accumulo/core/file/FileSKVWriter.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/file/FileSKVWriter.java b/core/src/main/java/org/apache/accumulo/core/file/FileSKVWriter.java index f4aa888..98a366f 100644 --- a/core/src/main/java/org/apache/accumulo/core/file/FileSKVWriter.java +++ b/core/src/main/java/org/apache/accumulo/core/file/FileSKVWriter.java @@ -36,4 +36,6 @@ public interface FileSKVWriter { DataOutputStream createMetaStore(String name) throws IOException; void close() throws IOException; + + long getLength() throws IOException; } http://git-wip-us.apache.org/repos/asf/accumulo/blob/65032235/core/src/main/java/org/apache/accumulo/core/file/blockfile/BlockFileWriter.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/file/blockfile/BlockFileWriter.java b/core/src/main/java/org/apache/accumulo/core/file/blockfile/BlockFileWriter.java index 570a8a5..e9d97c5 100644 --- a/core/src/main/java/org/apache/accumulo/core/file/blockfile/BlockFileWriter.java +++ b/core/src/main/java/org/apache/accumulo/core/file/blockfile/BlockFileWriter.java @@ -33,4 +33,6 @@ public interface BlockFileWriter { ABlockWriter prepareDataBlock() throws IOException; void close() throws IOException; + + long getLength() throws IOException; } http://git-wip-us.apache.org/repos/asf/accumulo/blob/65032235/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java b/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java index d00a797..8a6f38b 100644 --- a/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java +++ b/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java @@ -55,7 +55,8 @@ public class CachableBlockFile { public static class Writer implements BlockFileWriter { private BCFile.Writer _bc; private BlockWrite _bw; - private FSDataOutputStream fsout = null; + private final FSDataOutputStream fsout; + private long length = 0; public Writer(FileSystem fs, Path fName, String compressAlgor, Configuration conf, AccumuloConfiguration accumuloConfiguration) throws IOException { this.fsout = fs.create(fName); @@ -89,10 +90,13 @@ public class CachableBlockFile { _bw.close(); _bc.close(); - if (this.fsout != null) { - this.fsout.close(); - } + length = this.fsout.getPos(); + this.fsout.close(); + } + @Override + public long getLength() throws IOException { + return length; } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/65032235/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java index 060d956..54b01b4 100644 --- a/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java +++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java @@ -303,6 +303,7 @@ public class RFile { private boolean startedDefaultLocalityGroup = false; private HashSet previousColumnFamilies; + private long length = -1; public Writer(BlockFileWriter bfw, int blockSize) throws IOException { this(bfw, blockSize, (int) AccumuloConfiguration.getDefaultConfiguration().getMemoryInBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX)); @@ -340,8 +341,8 @@ public class RFile { } mba.close(); - fileWriter.close(); + length = fileWriter.getLength(); closed = true; } @@ -465,6 +466,14 @@ public class RFile { public boolean supportsLocalityGroups() { return true; } + + @Override + public long getLength() throws IOException { + if (!closed) { + return fileWriter.getLength(); + } + return length; + } } private static class LocalityGroupReader extends LocalityGroup implements FileSKVIterator { http://git-wip-us.apache.org/repos/asf/accumulo/blob/65032235/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Compactor.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Compactor.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Compactor.java index 0a8a9e3..c1b8105 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Compactor.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Compactor.java @@ -219,19 +219,16 @@ public class Compactor implements Callable { mfw = null; // set this to null so we do not try to close it again in finally if the close fails mfwTmp.close(); // if the close fails it will cause the compaction to fail - // Verify the file, since hadoop 0.20.2 sometimes lies about the success of close() - try { - FileSKVIterator openReader = fileFactory.openReader(outputFile.path().toString(), false, ns, ns.getConf(), acuTableConf); - openReader.close(); - } catch (IOException ex) { - log.error("Verification of successful compaction fails!!! " + extent + " " + outputFile, ex); - throw ex; - } - - log.debug(String.format("Compaction %s %,d read | %,d written | %,6d entries/sec | %6.3f secs", extent, majCStats.getEntriesRead(), - majCStats.getEntriesWritten(), (int) (majCStats.getEntriesRead() / ((t2 - t1) / 1000.0)), (t2 - t1) / 1000.0)); - - majCStats.setFileSize(fileFactory.getFileSize(outputFile.path().toString(), ns, ns.getConf(), acuTableConf)); + log.debug(String.format("Compaction %s %,d read | %,d written | %,6d entries/sec | %,6.3f secs | %,12d bytes | %9.3f byte/sec", + extent, + majCStats.getEntriesRead(), + majCStats.getEntriesWritten(), + (int) (majCStats.getEntriesRead() / ((t2 - t1) / 1000.0)), + (t2 - t1) / 1000.0, + mfwTmp.getLength(), + mfwTmp.getLength() / ((t2 - t1) / 1000.0))); + + majCStats.setFileSize(mfwTmp.getLength()); return majCStats; } catch (IOException e) { log.error("{}", e.getMessage(), e);