drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From par...@apache.org
Subject drill git commit: DRILL-3557: Ensure empty CSV's path can be added
Date Wed, 05 Aug 2015 16:57:02 GMT
Repository: drill
Updated Branches:
  refs/heads/master 974c6f2bf -> a96f793d7


DRILL-3557: Ensure empty CSV's path can be added


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/a96f793d
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/a96f793d
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/a96f793d

Branch: refs/heads/master
Commit: a96f793d7080e15041ac148241ac77d827bf2e6b
Parents: 974c6f2
Author: Hsuan-Yi Chu <hsuanyi@usc.edu>
Authored: Mon Jul 27 22:54:20 2015 -0700
Committer: Parth Chandra <parthc@apache.org>
Committed: Wed Aug 5 09:56:35 2015 -0700

----------------------------------------------------------------------
 .../drill/exec/store/schedule/BlockMapBuilder.java    | 14 ++++++++++++++
 .../java/org/apache/drill/TestExampleQueries.java     | 14 ++++++++++++--
 .../store/text/directoryWithEmpyCSV/empty.csv         |  0
 3 files changed, 26 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/a96f793d/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/BlockMapBuilder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/BlockMapBuilder.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/BlockMapBuilder.java
index e233dda..c5c351e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/BlockMapBuilder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/BlockMapBuilder.java
@@ -84,6 +84,12 @@ public class BlockMapBuilder {
 
   private class BlockMapReader extends TimedRunnable<List<CompleteFileWork>>
{
     final FileStatus status;
+
+    // This variable blockify indicates if a single file can be read by multiple threads
+    // For examples, for CSV, it is set as true
+    // because each row in a CSV file can be considered as an independent record;
+    // for json, it is set as false
+    // because each row in a json file cannot be determined as a record or not simply by
that row alone
     final boolean blockify;
 
     public BlockMapReader(FileStatus status, boolean blockify) {
@@ -114,6 +120,14 @@ public class BlockMapBuilder {
         work.add(new CompleteFileWork(getEndpointByteMap(new FileStatusWork(status)), 0,
status.getLen(), status.getPath().toString()));
       }
 
+      // This if-condition is specific for empty CSV file
+      // For CSV files, the global variable blockify is set as true
+      // And if this CSV file is empty, rangeMap would be empty also
+      // Therefore, at the point before this if-condition, work would not be populated
+      if(work.isEmpty()) {
+        work.add(new CompleteFileWork(getEndpointByteMap(new FileStatusWork(status)), 0,
0, status.getPath().toString()));
+      }
+
       return work;
     }
 

http://git-wip-us.apache.org/repos/asf/drill/blob/a96f793d/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
index e8af325..6b74ecf 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
@@ -1069,7 +1069,7 @@ public class TestExampleQueries extends BaseTestQuery {
         .unOrdered()
         .sqlBaselineQuery(baseQuery)
         .build()
-        .run();;
+        .run();
 
     // JoinQuery: star + window function
     final String joinQuery =
@@ -1088,7 +1088,17 @@ public class TestExampleQueries extends BaseTestQuery {
         .sqlBaselineQuery(joinBaseQuery)
         .build()
         .run();
-
   }
 
+  @Test // see DRILL-3557
+  public void testEmptyCSVinDirectory() throws Exception {
+    final String root = FileUtils.getResourceAsFile("/store/text/directoryWithEmpyCSV").toURI().toString();
+    final String toFile = FileUtils.getResourceAsFile("/store/text/directoryWithEmpyCSV/empty.csv").toURI().toString();
+
+    String query1 = String.format("explain plan for select * from dfs_test.`%s`", root);
+    String query2 = String.format("explain plan for select * from dfs_test.`%s`", toFile);
+
+    test(query1);
+    test(query2);
+  }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/a96f793d/exec/java-exec/src/test/resources/store/text/directoryWithEmpyCSV/empty.csv
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/store/text/directoryWithEmpyCSV/empty.csv b/exec/java-exec/src/test/resources/store/text/directoryWithEmpyCSV/empty.csv
new file mode 100644
index 0000000..e69de29


Mime
View raw message