crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jwi...@apache.org
Subject git commit: CRUNCH-408: Fix the HFileSource globStatus check to work on Hadoop2
Date Mon, 02 Jun 2014 03:45:00 GMT
Repository: crunch
Updated Branches:
  refs/heads/apache-crunch-0.8 c4df463ee -> 9a0c9e995


CRUNCH-408: Fix the HFileSource globStatus check to work on Hadoop2


Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/9a0c9e99
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/9a0c9e99
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/9a0c9e99

Branch: refs/heads/apache-crunch-0.8
Commit: 9a0c9e99546eff051033274b773a6c0925f30177
Parents: c4df463
Author: Josh Wills <jwills@apache.org>
Authored: Sun Jun 1 20:41:01 2014 -0700
Committer: Josh Wills <jwills@apache.org>
Committed: Sun Jun 1 20:43:04 2014 -0700

----------------------------------------------------------------------
 .../org/apache/crunch/io/hbase/HFileSource.java | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/crunch/blob/9a0c9e99/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileSource.java
----------------------------------------------------------------------
diff --git a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileSource.java b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileSource.java
index 8b5dab5..fe8d107 100644
--- a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileSource.java
+++ b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileSource.java
@@ -102,10 +102,28 @@ public class HFileSource extends FileSourceImpl<KeyValue> implements
ReadableSou
     long sum = 0;
     for (Path path : getPaths()) {
       try {
-        sum += SourceTargetHelper.getPathSize(conf, new Path(path, "*"));
+        sum += getSizeInternal(conf, path);
       } catch (IOException e) {
         LOG.warn("Failed to estimate size of " + path);
       }
+      System.out.println("Size after read of path = " + path.toString() + " = " + sum);
+    }
+    return sum;
+  }
+
+  private long getSizeInternal(Configuration conf, Path path) throws IOException {
+    FileSystem fs = path.getFileSystem(conf);
+    FileStatus[] statuses = fs.globStatus(path, HFileInputFormat.HIDDEN_FILE_FILTER);
+    if (statuses == null) {
+      return 0;
+    }
+    long sum = 0;
+    for (FileStatus status : statuses) {
+      if (status.isDir()) {
+        sum += SourceTargetHelper.getPathSize(fs, status.getPath());
+      } else {
+        sum += status.getLen();
+      }
     }
     return sum;
   }


Mime
View raw message