incubator-crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jwi...@apache.org
Subject git commit: CRUNCH-131: Use globStatus in SourceTargetHelper so that wildcard patterns work properly with Crunch. Contributed by Dave Beech.
Date Thu, 13 Dec 2012 19:16:25 GMT
Updated Branches:
  refs/heads/master 092ef5d01 -> 4a3aa0dfd


CRUNCH-131: Use globStatus in SourceTargetHelper so that wildcard patterns work properly with
Crunch. Contributed by Dave Beech.


Project: http://git-wip-us.apache.org/repos/asf/incubator-crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-crunch/commit/4a3aa0df
Tree: http://git-wip-us.apache.org/repos/asf/incubator-crunch/tree/4a3aa0df
Diff: http://git-wip-us.apache.org/repos/asf/incubator-crunch/diff/4a3aa0df

Branch: refs/heads/master
Commit: 4a3aa0dfd9dbdfd2532b03776d87b99f71fae5a5
Parents: 092ef5d
Author: Josh Wills <jwills@apache.org>
Authored: Thu Dec 13 11:15:50 2012 -0800
Committer: Josh Wills <jwills@apache.org>
Committed: Thu Dec 13 11:15:50 2012 -0800

----------------------------------------------------------------------
 .../org/apache/crunch/io/SourceTargetHelper.java   |   22 ++-------------
 1 files changed, 3 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/4a3aa0df/crunch/src/main/java/org/apache/crunch/io/SourceTargetHelper.java
----------------------------------------------------------------------
diff --git a/crunch/src/main/java/org/apache/crunch/io/SourceTargetHelper.java b/crunch/src/main/java/org/apache/crunch/io/SourceTargetHelper.java
index bbe5eaa..65fb35a 100644
--- a/crunch/src/main/java/org/apache/crunch/io/SourceTargetHelper.java
+++ b/crunch/src/main/java/org/apache/crunch/io/SourceTargetHelper.java
@@ -17,7 +17,6 @@
  */
 package org.apache.crunch.io;
 
-import java.io.FileNotFoundException;
 import java.io.IOException;
 
 import org.apache.commons.logging.Log;
@@ -40,28 +39,13 @@ public class SourceTargetHelper {
   }
 
   public static long getPathSize(FileSystem fs, Path path) throws IOException {
-
-    if (!fs.exists(path)) {
+    FileStatus[] stati = fs.globStatus(path);
+    if (stati == null || stati.length == 0) {
       return -1L;
     }
-
-    FileStatus[] stati = null;
-    try {
-      stati = fs.listStatus(path);
-      if (stati == null) {
-        throw new FileNotFoundException(path + " doesn't exist");
-      }
-    } catch (FileNotFoundException e) {
-      LOG.warn("Returning 0 for getPathSize on non-existant path '" + path + "'");
-      return 0L;
-    }
-
-    if (stati.length == 0) {
-      return 0L;
-    }
     long size = 0;
     for (FileStatus status : stati) {
-      size += status.getLen();
+      size += fs.getContentSummary(status.getPath()).getLength();
     }
     return size;
   }


Mime
View raw message