drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From meh...@apache.org
Subject [1/3] drill git commit: DRILL-3965: Fix IOOB while trying to perform partition pruning
Date Fri, 23 Oct 2015 06:28:55 GMT
Repository: drill
Updated Branches:
  refs/heads/master 19b4b798b -> a98da39dd


DRILL-3965: Fix IOOB while trying to perform partition pruning


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/a98da39d
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/a98da39d
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/a98da39d

Branch: refs/heads/master
Commit: a98da39dd5a8fa368afd8765f4e981826bbfcc0f
Parents: 17abf36
Author: Mehant Baid <mehantr@gmail.com>
Authored: Mon Oct 19 23:49:45 2015 -0700
Committer: Mehant Baid <mehantr@gmail.com>
Committed: Thu Oct 22 23:14:27 2015 -0700

----------------------------------------------------------------------
 .../planner/ParquetPartitionDescriptor.java     |  2 +-
 .../exec/planner/ParquetPartitionLocation.java  | 54 ++++++++++++++++++++
 .../apache/drill/TestCTASPartitionFilter.java   |  7 +++
 3 files changed, 62 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/a98da39d/exec/java-exec/src/main/java/org/apache/drill/exec/planner/ParquetPartitionDescriptor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/ParquetPartitionDescriptor.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/ParquetPartitionDescriptor.java
index 880daa6..5294b30 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/ParquetPartitionDescriptor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/ParquetPartitionDescriptor.java
@@ -123,7 +123,7 @@ public class ParquetPartitionDescriptor extends AbstractPartitionDescriptor
{
     Set<String> fileLocations = ((ParquetGroupScan) scanRel.getGroupScan()).getFileSet();
     List<PartitionLocation> locations = new LinkedList<>();
     for (String file: fileLocations) {
-      locations.add(new DFSPartitionLocation(MAX_NESTED_SUBDIRS, getBaseTableLocation(),
file));
+      locations.add(new ParquetPartitionLocation(file));
     }
     locationSuperList = Lists.partition(locations, PartitionDescriptor.PARTITION_BATCH_SIZE);
     sublistsCreated = true;

http://git-wip-us.apache.org/repos/asf/drill/blob/a98da39d/exec/java-exec/src/main/java/org/apache/drill/exec/planner/ParquetPartitionLocation.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/ParquetPartitionLocation.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/ParquetPartitionLocation.java
new file mode 100644
index 0000000..719b080
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/ParquetPartitionLocation.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner;
+
+/*
+ * PartitionLocation for the parquet auto partitioning scheme. We just store
+ * the location of each partition within this class. Since the partition value
+ * is obtained from the file metadata and not from the location string (like in directory
based
+ * partitioning scheme) we throw UnsupportedOperationException when getPartitionValue() is
+ * invoked.
+ */
+public class ParquetPartitionLocation implements PartitionLocation {
+  private final String file;
+
+  public ParquetPartitionLocation(String file) {
+    this.file = file;
+  }
+
+  /**
+   * Parquet CTAS auto partitioning scheme does not support getting the partition value
+   * based on the location string.
+   * @param index
+   * @return
+   */
+  @Override
+  public String getPartitionValue(int index) {
+    throw new UnsupportedOperationException("Getting partitioning column value from the partition
location is not " +
+        "supported by parquet auto partitioning scheme");
+  }
+
+  /**
+   * Get the location of this partition
+   * @return String location of the partition
+   */
+  @Override
+  public String getEntirePartitionLocation() {
+    return file;
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/a98da39d/exec/java-exec/src/test/java/org/apache/drill/TestCTASPartitionFilter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestCTASPartitionFilter.java b/exec/java-exec/src/test/java/org/apache/drill/TestCTASPartitionFilter.java
index 9886024..1f49f74 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestCTASPartitionFilter.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestCTASPartitionFilter.java
@@ -48,6 +48,13 @@ public class TestCTASPartitionFilter extends PlanTestBase {
   }
 
   @Test
+  public void testDrill3965() throws Exception {
+    test("use dfs_test.tmp");
+    test("create table orders_auto_partition partition by(o_orderpriority) as select * from
cp.`tpch/orders.parquet`");
+    test("explain plan for select count(*) from `orders_auto_partition/1_0_1.parquet` where
o_orderpriority = '5-LOW'");
+  }
+
+  @Test
   public void withDistribution() throws Exception {
     test("alter session set `planner.slice_target` = 1");
     test("alter session set `store.partition.hash_distribute` = true");


Mime
View raw message