crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jwi...@apache.org
Subject git commit: CRUNCH-221: Ignore hidden files during materialization.
Date Wed, 19 Jun 2013 20:08:06 GMT
Updated Branches:
  refs/heads/master a97e8e101 -> 687894564


CRUNCH-221: Ignore hidden files during materialization.


Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/68789456
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/68789456
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/68789456

Branch: refs/heads/master
Commit: 6878945645bd15036bd15c1e4b5a17ab6691fc11
Parents: a97e8e1
Author: Josh Wills <jwills@apache.org>
Authored: Wed Jun 19 13:05:15 2013 -0700
Committer: Josh Wills <jwills@apache.org>
Committed: Wed Jun 19 13:05:15 2013 -0700

----------------------------------------------------------------------
 .../crunch/io/CompositePathIterableIT.java       | 19 +++++++++++++++++++
 .../apache/crunch/io/CompositePathIterable.java  |  3 ++-
 2 files changed, 21 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/crunch/blob/68789456/crunch-core/src/it/java/org/apache/crunch/io/CompositePathIterableIT.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/it/java/org/apache/crunch/io/CompositePathIterableIT.java b/crunch-core/src/it/java/org/apache/crunch/io/CompositePathIterableIT.java
index 08d226d..22f5a8e 100644
--- a/crunch-core/src/it/java/org/apache/crunch/io/CompositePathIterableIT.java
+++ b/crunch-core/src/it/java/org/apache/crunch/io/CompositePathIterableIT.java
@@ -81,4 +81,23 @@ public class CompositePathIterableIT {
         Writables.strings()));
   }
 
+  @Test
+  public void testCreate_HiddenFiles() throws IOException {
+    File file = tmpDir.copyResourceFile("set1.txt");
+    assertTrue(file.renameTo(new File(tmpDir.getRootFile(), "_set1.txt")));
+
+    file = tmpDir.copyResourceFile("set1.txt");
+    assertTrue(file.renameTo(new File(tmpDir.getRootFile(), ".set1.txt")));
+
+    Path emptyInputDir = tmpDir.getRootPath();
+
+    Configuration conf = new Configuration();
+    LocalFileSystem local = FileSystem.getLocal(conf);
+
+    Iterable<String> iterable = CompositePathIterable.create(local, emptyInputDir,
+        new TextFileReaderFactory<String>(Writables.strings()));
+
+    assertTrue(Lists.newArrayList(iterable).isEmpty());
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/crunch/blob/68789456/crunch-core/src/main/java/org/apache/crunch/io/CompositePathIterable.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/io/CompositePathIterable.java b/crunch-core/src/main/java/org/apache/crunch/io/CompositePathIterable.java
index a4723e9..c36a506 100644
--- a/crunch-core/src/main/java/org/apache/crunch/io/CompositePathIterable.java
+++ b/crunch-core/src/main/java/org/apache/crunch/io/CompositePathIterable.java
@@ -38,7 +38,8 @@ public class CompositePathIterable<T> implements Iterable<T>
{
   private static final PathFilter FILTER = new PathFilter() {
     @Override
     public boolean accept(Path path) {
-      return !path.getName().startsWith("_");
+      String name = path.getName();
+      return !name.startsWith("_") && !name.startsWith(".");
     }
   };
 


Mime
View raw message