drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s..@apache.org
Subject drill git commit: DRILL-2532: Fix glob implementation
Date Sat, 09 May 2015 22:15:39 GMT
Repository: drill
Updated Branches:
  refs/heads/master 4844b0e61 -> 76cf72969
Updated Tags:  refs/tags/0.9.0 [created] 78fd658d2
  refs/tags/drill-root-0.4.0-incubating [created] caa15e262


DRILL-2532: Fix glob implementation


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/76cf7296
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/76cf7296
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/76cf7296

Branch: refs/heads/master
Commit: 76cf72969014788a882204afd88fbaf944a981f9
Parents: 4844b0e
Author: AdamPD <adam@pharmadata.net.au>
Authored: Tue Mar 24 12:33:14 2015 +1000
Committer: Steven Phillips <smp@apache.org>
Committed: Sat May 9 13:45:01 2015 -0700

----------------------------------------------------------------------
 .../drill/exec/store/dfs/FileSelection.java     | 54 +++++++++-------
 .../apache/drill/exec/store/dfs/TestGlob.java   | 67 ++++++++++++++++++++
 2 files changed, 97 insertions(+), 24 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/76cf7296/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java
index b9ae303..be9784e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java
@@ -127,35 +127,41 @@ public class FileSelection {
     return statuses;
   }
 
-  public static FileSelection create(DrillFileSystem fs, String parent, String path) throws
IOException {
-    if ( !(path.contains("*") || path.contains("?")) ) {
-      Path p = new Path(parent, removeLeadingSlash(path));
-      FileStatus status = fs.getFileStatus(p);
-      return new FileSelection(Collections.singletonList(status), p.toUri().getPath());
-    } else {
-      Path p = new Path(parent,removeLeadingSlash(path));
-      FileStatus[] status = fs.globStatus(p);
-      if (status == null || status.length == 0) {
-        return null;
-      }
-      String[] s = p.toUri().getPath().split("/");
-      int i = 0;
-
-      // get a selection root based on the portions of the selection path that don't contain
a wildcard.
-      for(; i < s.length; i++){
-         if(s[i].contains("*") || s[i].contains("?")){
-           break;
-         }
+  public static String commonPath(FileStatus... paths){
+    String commonPath = "";
+    String[][] folders = new String[paths.length][];
+    for(int i = 0; i < paths.length; i++){
+      folders[i] = Path.getPathWithoutSchemeAndAuthority(paths[i].getPath()).toString().split("/");
+    }
+    for(int j = 0; j < folders[0].length; j++){
+      String thisFolder = folders[0][j];
+      boolean allMatched = true;
+      for(int i = 1; i < folders.length && allMatched; i++){
+        if(folders[i].length < j){
+          allMatched = false;
+          break;
+        }
+        allMatched &= folders[i][j].equals(thisFolder);
       }
-      String newPath;
-      if(i > 0){
-        newPath = StringUtils.join(ArrayUtils.subarray(s, 0, i), "/");
+      if(allMatched){
+        commonPath += thisFolder + "/";
       }else{
-        newPath = "/";
+        break;
       }
+    }
+    return commonPath;
+  }
 
-      return new FileSelection(Lists.newArrayList(status), newPath);
+  public static FileSelection create(DrillFileSystem fs, String parent, String path) throws
IOException {
+    Path p = new Path(parent,removeLeadingSlash(path));
+    FileStatus[] status = fs.globStatus(p);
+    if (status == null || status.length == 0) {
+      return null;
+    }
+    if (status.length == 1) {
+      return new FileSelection(Collections.singletonList(status[0]), p.toUri().getPath());
     }
+    return new FileSelection(Lists.newArrayList(status), commonPath(status));
   }
 
   private static String removeLeadingSlash(String path) {

http://git-wip-us.apache.org/repos/asf/drill/blob/76cf7296/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestGlob.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestGlob.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestGlob.java
new file mode 100644
index 0000000..8b4f7e7
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestGlob.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.dfs;
+
+import org.apache.drill.BaseTestQuery;
+import org.apache.drill.common.util.TestTools;
+import org.junit.Test;
+
+public class TestGlob extends BaseTestQuery {
+
+    String MULTILEVEL = TestTools.getWorkingPath() + "/../java-exec/src/test/resources/multilevel";
+
+    @Test
+    public void testGlobSet() throws Exception {
+        testBuilder()
+            .sqlQuery(String.format("select count(*) from dfs_test.`%s/parquet/{1994,1995}`",
MULTILEVEL))
+            .unOrdered()
+            .baselineColumns("EXPR$0")
+            .baselineValues(80L)
+            .build().run();
+    }
+
+    @Test
+    public void testGlobWildcard() throws Exception {
+        testBuilder()
+            .sqlQuery(String.format("select count(*) from dfs_test.`%s/parquet/1994/*`",
MULTILEVEL))
+            .unOrdered()
+            .baselineColumns("EXPR$0")
+            .baselineValues(40L)
+            .build().run();
+    }
+
+    @Test
+    public void testGlobSingleCharacter() throws Exception {
+        testBuilder()
+            .sqlQuery(String.format("select count(*) from dfs_test.`%s/parquet/199?/*`",
MULTILEVEL))
+            .unOrdered()
+            .baselineColumns("EXPR$0")
+            .baselineValues(120L)
+            .build().run();
+    }
+
+    @Test
+    public void testGlobSingleCharacterRange() throws Exception {
+        testBuilder()
+            .sqlQuery(String.format("select count(*) from dfs_test.`%s/parquet/199[4-5]/*`",
MULTILEVEL))
+            .unOrdered()
+            .baselineColumns("EXPR$0")
+            .baselineValues(80L)
+            .build().run();
+    }
+}


Mime
View raw message