hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vvasu...@apache.org
Subject [44/50] hadoop git commit: HADOOP-10798. globStatus() should always return a sorted list of files (cmccabe)
Date Wed, 01 Jul 2015 10:46:15 GMT
HADOOP-10798. globStatus() should always return a sorted list of files (cmccabe)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/68e588cb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/68e588cb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/68e588cb

Branch: refs/heads/YARN-2139
Commit: 68e588cbee660d55dba518892d064bee3795a002
Parents: d0cc038
Author: Colin Patrick Mccabe <cmccabe@cloudera.com>
Authored: Tue Jun 30 16:40:21 2015 -0700
Committer: Colin Patrick Mccabe <cmccabe@cloudera.com>
Committed: Tue Jun 30 16:40:21 2015 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt |  3 +++
 .../main/java/org/apache/hadoop/fs/Globber.java | 11 ++++++++-
 .../org/apache/hadoop/fs/TestGlobPaths.java     | 25 ++++++++++++++++++++
 3 files changed, 38 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/68e588cb/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 2aab8b4..4d69a229 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -905,6 +905,9 @@ Release 2.8.0 - UNRELEASED
     HADOOP-12154. FileSystem#getUsed() returns the file length only from root '/'
     (J.Andreina via vinayakumarb)
 
+    HADOOP-10798. globStatus() should always return a sorted list of files
+    (cmccabe)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/68e588cb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
index 8a8137a..9cb810f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.fs;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 
 import org.apache.commons.logging.LogFactory;
@@ -285,6 +286,14 @@ class Globber {
         (flattenedPatterns.size() <= 1)) {
       return null;
     }
-    return results.toArray(new FileStatus[0]);
+    /*
+     * In general, the results list will already be sorted, since listStatus
+     * returns results in sorted order for many Hadoop filesystems.  However,
+     * not all Hadoop filesystems have this property.  So we sort here in order
+     * to get consistent results.  See HADOOP-10798 for details.
+     */
+    FileStatus ret[] = results.toArray(new FileStatus[0]);
+    Arrays.sort(ret);
+    return ret;
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/68e588cb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java
index 50e2e5b..afd8fb2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java
@@ -21,9 +21,11 @@ import static org.junit.Assert.*;
 
 import java.io.IOException;
 import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
 import java.util.UUID;
 import java.util.regex.Pattern;
 
+import com.google.common.collect.Ordering;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.permission.FsPermission;
@@ -1284,4 +1286,27 @@ public class TestGlobPaths {
   public void testNonTerminalGlobsOnFC() throws Exception {
     testOnFileContext(new TestNonTerminalGlobs(true));
   }
+
+  @Test
+  public void testLocalFilesystem() throws Exception {
+    Configuration conf = new Configuration();
+    FileSystem fs = FileSystem.getLocal(conf);
+    String localTmp = System.getProperty("java.io.tmpdir");
+    Path base = new Path(new Path(localTmp), UUID.randomUUID().toString());
+    Assert.assertTrue(fs.mkdirs(base));
+    Assert.assertTrue(fs.mkdirs(new Path(base, "e")));
+    Assert.assertTrue(fs.mkdirs(new Path(base, "c")));
+    Assert.assertTrue(fs.mkdirs(new Path(base, "a")));
+    Assert.assertTrue(fs.mkdirs(new Path(base, "d")));
+    Assert.assertTrue(fs.mkdirs(new Path(base, "b")));
+    fs.deleteOnExit(base);
+    FileStatus[] status = fs.globStatus(new Path(base, "*"));
+    ArrayList list = new ArrayList();
+    for (FileStatus f: status) {
+        list.add(f.getPath().toString());
+    }
+    boolean sorted = Ordering.natural().isOrdered(list);
+    Assert.assertTrue(sorted);
+  }
 }
+


Mime
View raw message