crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jwi...@apache.org
Subject git commit: CRUNCH-234: Fix non-zero size on empty intermediate outputs in hadoop2
Date Mon, 08 Jul 2013 18:05:34 GMT
Updated Branches:
  refs/heads/master 045557973 -> 37a73f103


CRUNCH-234: Fix non-zero size on empty intermediate outputs in hadoop2


Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/37a73f10
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/37a73f10
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/37a73f10

Branch: refs/heads/master
Commit: 37a73f1039ddb205874acb58177772cef86e6485
Parents: 0455579
Author: Josh Wills <jwills@apache.org>
Authored: Mon Jul 8 09:51:42 2013 -0700
Committer: Josh Wills <jwills@apache.org>
Committed: Mon Jul 8 09:51:42 2013 -0700

----------------------------------------------------------------------
 .../main/java/org/apache/crunch/io/SourceTargetHelper.java   | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/crunch/blob/37a73f10/crunch-core/src/main/java/org/apache/crunch/io/SourceTargetHelper.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/io/SourceTargetHelper.java b/crunch-core/src/main/java/org/apache/crunch/io/SourceTargetHelper.java
index 66764cf..7b9bea0 100644
--- a/crunch-core/src/main/java/org/apache/crunch/io/SourceTargetHelper.java
+++ b/crunch-core/src/main/java/org/apache/crunch/io/SourceTargetHelper.java
@@ -41,7 +41,13 @@ public class SourceTargetHelper {
     }
     long size = 0;
     for (FileStatus status : stati) {
-      size += fs.getContentSummary(status.getPath()).getLength();
+      if (status.isDir()) {
+        for (FileStatus st : fs.listStatus(status.getPath())) {
+          size += st.getLen();
+        }
+      } else {
+        size += status.getLen();
+      }
     }
     return size;
   }


Mime
View raw message