Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 3173910546 for ; Wed, 13 Nov 2013 17:48:09 +0000 (UTC) Received: (qmail 24189 invoked by uid 500); 13 Nov 2013 17:48:03 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 24149 invoked by uid 500); 13 Nov 2013 17:48:01 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 24071 invoked by uid 99); 13 Nov 2013 17:47:41 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 13 Nov 2013 17:47:41 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id D4F8B8A81B4; Wed, 13 Nov 2013 17:47:40 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: ecn@apache.org To: commits@accumulo.apache.org Message-Id: <7729686cf2294cc28878ab95abca3d72@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: git commit: ACCUMULO-1867 use the correct file system when reading the indexes of existing files Date: Wed, 13 Nov 2013 17:47:40 +0000 (UTC) Updated Branches: refs/heads/1.6.0-SNAPSHOT ff9bfeb88 -> 5ded01923 ACCUMULO-1867 use the correct file system when reading the indexes of existing files Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/5ded0192 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/5ded0192 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/5ded0192 Branch: refs/heads/1.6.0-SNAPSHOT Commit: 5ded019232528880fa1746134799571abde19a85 Parents: ff9bfeb Author: Eric Newton Authored: Wed Nov 13 12:47:51 2013 -0500 Committer: Eric Newton Committed: Wed Nov 13 12:47:59 2013 -0500 ---------------------------------------------------------------------- .../apache/accumulo/server/ServerConstants.java | 5 +++ .../apache/accumulo/server/util/FileUtil.java | 39 ++++++++++---------- 2 files changed, 25 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/5ded0192/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java index 9e0ac39..dfb312c 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java +++ b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java @@ -134,4 +134,9 @@ public class ServerConstants { public static String[] getMetadataTableDirs() { return prefix(getTablesDirs(), MetadataTable.ID); } + + public static String[] getTemporaryDirs() { + return prefix(getBaseDirs(), "tmp"); + } + } http://git-wip-us.apache.org/repos/asf/accumulo/blob/5ded0192/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java index fa13f1b..7e20853 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java @@ -44,6 +44,7 @@ import org.apache.accumulo.core.iterators.SortedKeyValueIterator; import org.apache.accumulo.core.iterators.system.MultiIterator; import org.apache.accumulo.core.util.CachedConfiguration; import org.apache.accumulo.core.util.LocalityGroupUtil; +import org.apache.accumulo.server.ServerConstants; import org.apache.accumulo.server.fs.FileRef; import org.apache.accumulo.server.fs.VolumeManager; import org.apache.hadoop.conf.Configuration; @@ -75,33 +76,32 @@ public class FileUtil { private static final Logger log = Logger.getLogger(FileUtil.class); - private static String createTmpDir(AccumuloConfiguration acuConf, VolumeManager fs) throws IOException { - String accumuloDir = acuConf.get(Property.INSTANCE_DFS_DIR); + private static Path createTmpDir(AccumuloConfiguration acuConf, VolumeManager fs) throws IOException { + String accumuloDir = fs.choose(ServerConstants.getTemporaryDirs()); - String tmpDir = null; - while (tmpDir == null) { - tmpDir = accumuloDir + "/tmp/idxReduce_" + String.format("%09d", (int) (Math.random() * Integer.MAX_VALUE)); + Path result = null; + while (result == null) { + result = new Path(accumuloDir + "/tmp/idxReduce_" + String.format("%09d", (int) (Math.random() * Integer.MAX_VALUE))); try { - fs.getFileStatus(new Path(tmpDir)); - tmpDir = null; + fs.getFileStatus(result); + result = null; continue; } catch (FileNotFoundException fne) { // found an unused temp directory } - fs.mkdirs(new Path(tmpDir)); + fs.mkdirs(result); // try to reserve the tmp dir - if (!fs.createNewFile(new Path(tmpDir + "/__reserve"))) - tmpDir = null; + if (!fs.createNewFile(new Path(result, "__reserve"))) + result = null; } - - return tmpDir; + return result; } public static Collection reduceFiles(AccumuloConfiguration acuConf, Configuration conf, VolumeManager fs, Text prevEndRow, Text endRow, - Collection mapFiles, int maxFiles, String tmpDir, int pass) throws IOException { + Collection mapFiles, int maxFiles, Path tmpDir, int pass) throws IOException { ArrayList paths = new ArrayList(mapFiles); if (paths.size() <= maxFiles) @@ -121,7 +121,7 @@ public class FileUtil { start = end; - FileRef newMapFile = new FileRef(String.format("%s/%04d." + RFile.EXTENSION, newDir, count++)); + FileRef newMapFile = new FileRef(String.format("%s/%04d.%s", newDir, count++, RFile.EXTENSION)); outFiles.add(newMapFile); FileSystem ns = fs.getFileSystemByPath(newMapFile.path()); @@ -132,6 +132,7 @@ public class FileUtil { FileSKVIterator reader = null; try { for (FileRef s : inFiles) { + ns = fs.getFileSystemByPath(s.path()); reader = FileOperations.getInstance().openIndex(s.path().toString(), ns, ns.getConf(), acuConf); iters.add(reader); } @@ -192,7 +193,7 @@ public class FileUtil { Configuration conf = CachedConfiguration.getInstance(); - String tmpDir = null; + Path tmpDir = null; int maxToOpen = acuconf.getCount(Property.TSERV_TABLET_SPLIT_FINDMIDPOINT_MAXOPEN); ArrayList readers = new ArrayList(mapFiles.size()); @@ -265,7 +266,7 @@ public class FileUtil { Collection origMapFiles = mapFiles; - String tmpDir = null; + Path tmpDir = null; int maxToOpen = acuConf.getCount(Property.TSERV_TABLET_SPLIT_FINDMIDPOINT_MAXOPEN); ArrayList readers = new ArrayList(mapFiles.size()); @@ -359,7 +360,7 @@ public class FileUtil { } } - private static void cleanupIndexOp(AccumuloConfiguration acuConf, String tmpDir, VolumeManager fs, ArrayList readers) throws IOException { + private static void cleanupIndexOp(AccumuloConfiguration acuConf, Path tmpDir, VolumeManager fs, ArrayList readers) throws IOException { // close all of the index sequence files for (FileSKVIterator r : readers) { try { @@ -373,8 +374,8 @@ public class FileUtil { if (tmpDir != null) { String tmpPrefix = acuConf.get(Property.INSTANCE_DFS_DIR) + "/tmp"; - if (tmpDir.startsWith(tmpPrefix)) - fs.deleteRecursively(new Path(tmpDir)); + if (tmpDir.toUri().getPath().startsWith(tmpPrefix)) + fs.deleteRecursively(tmpDir); else log.error("Did not delete tmp dir because it wasn't a tmp dir " + tmpDir); }