falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rostafiyc...@apache.org
Subject falcon git commit: FALCON-1094 getAllFilesRecursivelyHDFS without recursive call. Contributed by Ruslan Ostafiychuk
Date Fri, 13 Mar 2015 14:00:44 GMT
Repository: falcon
Updated Branches:
  refs/heads/master 334669d84 -> 1d7ca192b


FALCON-1094 getAllFilesRecursivelyHDFS without recursive call. Contributed by Ruslan Ostafiychuk


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/1d7ca192
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/1d7ca192
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/1d7ca192

Branch: refs/heads/master
Commit: 1d7ca192b8ba88078e1716f4dc0ab81707c38144
Parents: 334669d
Author: Ruslan Ostafiychuk <rostafiychuk@apache.org>
Authored: Fri Mar 13 15:59:24 2015 +0200
Committer: Ruslan Ostafiychuk <rostafiychuk@apache.org>
Committed: Fri Mar 13 15:59:24 2015 +0200

----------------------------------------------------------------------
 falcon-regression/CHANGES.txt                   |  2 ++
 .../falcon/regression/core/util/HadoopUtil.java | 22 ++++++++------------
 2 files changed, 11 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/1d7ca192/falcon-regression/CHANGES.txt
----------------------------------------------------------------------
diff --git a/falcon-regression/CHANGES.txt b/falcon-regression/CHANGES.txt
index 4b38ad6..3e32d78 100644
--- a/falcon-regression/CHANGES.txt
+++ b/falcon-regression/CHANGES.txt
@@ -60,6 +60,8 @@ Trunk (Unreleased)
    via Samarth Gupta)
 
   IMPROVEMENTS
+   FALCON-1094 getAllFilesRecursivelyHDFS without recursive call (Ruslan Ostafiychuk)
+
    FALCON-1093 Tag all new tests added to falcon-regression (Paul Isaychuk via Ruslan Ostafiychuk)
 
    FALCON-1089 ProcessInstanceStatusTest improvement (Paul Isaychuk via Ruslan Ostafiychuk)

http://git-wip-us.apache.org/repos/asf/falcon/blob/1d7ca192/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java
b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java
index 7122274..e2c3ae2 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java
@@ -24,7 +24,9 @@ import org.apache.falcon.regression.core.helpers.ColoHelper;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.log4j.Logger;
 
 import java.io.File;
@@ -114,23 +116,17 @@ public final class HadoopUtil {
     public static List<Path> getAllFilesRecursivelyHDFS(
         FileSystem fs, Path location) throws IOException {
         List<Path> returnList = new ArrayList<Path>();
-        FileStatus[] stats;
+        RemoteIterator<LocatedFileStatus> remoteIterator;
         try {
-            stats = fs.listStatus(location);
+            remoteIterator = fs.listFiles(location, true);
         } catch (FileNotFoundException e) {
-            e.printStackTrace();
-            return new ArrayList<Path>();
-        }
-        if (stats == null) {
+            LOGGER.info("Path '" + location + "' is not found on " + fs.getUri());
             return returnList;
         }
-        for (FileStatus stat : stats) {
-            if (!isDir(stat)) {
-                if (!stat.getPath().toUri().toString().contains("_SUCCESS")) {
-                    returnList.add(stat.getPath());
-                }
-            } else {
-                returnList.addAll(getAllFilesRecursivelyHDFS(fs, stat.getPath()));
+        while(remoteIterator.hasNext()) {
+            Path path = remoteIterator.next().getPath();
+            if (!path.toUri().toString().contains("_SUCCESS")) {
+                returnList.add(path);
             }
         }
         return returnList;


Mime
View raw message