drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amansi...@apache.org
Subject drill git commit: DRILL-4147: Change union-all's output distribution trait to ANY.
Date Tue, 02 Aug 2016 05:56:21 GMT
Repository: drill
Updated Branches:
  refs/heads/master 4e1bdac08 -> 0700c6b10


DRILL-4147: Change union-all's output distribution trait to ANY.

Additional unit tests.

Address review comments.

close apache/drill#555


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/0700c6b1
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/0700c6b1
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/0700c6b1

Branch: refs/heads/master
Commit: 0700c6b100367feef514a833e1aeb914e4fe64c4
Parents: 4e1bdac
Author: Aman Sinha <asinha@maprtech.com>
Authored: Tue Dec 1 16:46:51 2015 -0800
Committer: Aman Sinha <asinha@maprtech.com>
Committed: Mon Aug 1 22:53:20 2016 -0700

----------------------------------------------------------------------
 .../exec/planner/physical/UnionAllPrule.java    |  6 +-
 .../java/org/apache/drill/TestUnionAll.java     | 98 ++++++++++++++++++++
 .../org/apache/drill/TestUnionDistinct.java     | 36 ++++++-
 3 files changed, 138 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/0700c6b1/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrule.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrule.java
index 51cf866..95feddd 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrule.java
@@ -61,7 +61,11 @@ public class UnionAllPrule extends Prule {
         convertedInputList.add(convertedInput);
       }
 
-      traits = call.getPlanner().emptyTraitSet().plus(Prel.DRILL_PHYSICAL).plus(DrillDistributionTrait.SINGLETON);
+      // output distribution trait is set to ANY since union-all inputs may be distributed
in different ways
+      // and unlike a join there are no join keys that allow determining how the output would
be distributed.
+      // Note that a downstream operator may impose a required distribution which would be
satisfied by
+      // inserting an Exchange after the Union-All.
+      traits = call.getPlanner().emptyTraitSet().plus(Prel.DRILL_PHYSICAL).plus(DrillDistributionTrait.ANY);
 
       Preconditions.checkArgument(convertedInputList.size() >= 2, "Union list must be
at least two items.");
       RelNode left = convertedInputList.get(0);

http://git-wip-us.apache.org/repos/asf/drill/blob/0700c6b1/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java b/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java
index 7092a4f..e4168c6 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java
@@ -18,6 +18,7 @@
 package org.apache.drill;
 
 import com.google.common.collect.Lists;
+
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.drill.common.exceptions.UserException;
 import org.apache.drill.common.expression.SchemaPath;
@@ -32,6 +33,9 @@ import java.util.List;
 public class TestUnionAll extends BaseTestQuery{
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestUnionAll.class);
 
+  private static final String sliceTargetSmall = "alter session set `planner.slice_target`
= 1";
+  private static final String sliceTargetDefault = "alter session reset `planner.slice_target`";
+
   @Test  // Simple Union-All over two scans
   public void testUnionAll1() throws Exception {
     String query = "(select n_regionkey from cp.`tpch/nation.parquet`) union all (select
r_regionkey from cp.`tpch/region.parquet`)";
@@ -724,6 +728,7 @@ public class TestUnionAll extends BaseTestQuery{
                     ".*Scan.*columns=\\[`columns`\\[0\\]\\].*\n" +
             ".*Project.*\n" +
                 ".*Scan.*columns=\\[`columns`\\[0\\]\\].*"};
+
     final String[] excludedPlan = {};
     PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan);
 
@@ -1013,4 +1018,97 @@ public class TestUnionAll extends BaseTestQuery{
         .build()
         .run();
   }
+
+  @Test // DRILL-4147 // base case
+  public void testDrill4147_1() throws Exception {
+    final String l = FileUtils.getResourceAsFile("/multilevel/parquet/1994").toURI().toString();
+    final String r = FileUtils.getResourceAsFile("/multilevel/parquet/1995").toURI().toString();
+
+    final String query = String.format("SELECT o_custkey FROM dfs_test.`%s` \n" +
+        "Union All SELECT o_custkey FROM dfs_test.`%s`", l, r);
+
+    // Validate the plan
+    final String[] expectedPlan = {"UnionExchange.*\n",
+        ".*Project.*\n" +
+        ".*UnionAll"};
+    final String[] excludedPlan = {};
+
+    try {
+      test(sliceTargetSmall);
+      PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan);
+
+      testBuilder()
+        .optionSettingQueriesForTestQuery(sliceTargetSmall)
+        .optionSettingQueriesForBaseline(sliceTargetDefault)
+        .unOrdered()
+        .sqlQuery(query)
+        .sqlBaselineQuery(query)
+        .build()
+        .run();
+    } finally {
+      test(sliceTargetDefault);
+    }
+  }
+
+  @Test // DRILL-4147  // group-by on top of union-all
+  public void testDrill4147_2() throws Exception {
+    final String l = FileUtils.getResourceAsFile("/multilevel/parquet/1994").toURI().toString();
+    final String r = FileUtils.getResourceAsFile("/multilevel/parquet/1995").toURI().toString();
+
+    final String query = String.format("Select o_custkey, count(*) as cnt from \n" +
+        " (SELECT o_custkey FROM dfs_test.`%s` \n" +
+        "Union All SELECT o_custkey FROM dfs_test.`%s`) \n" +
+        "group by o_custkey", l, r);
+
+    // Validate the plan
+    final String[] expectedPlan = {"(?s)UnionExchange.*HashAgg.*HashToRandomExchange.*UnionAll.*"};
+    final String[] excludedPlan = {};
+
+    try {
+      test(sliceTargetSmall);
+      PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan);
+
+      testBuilder()
+        .optionSettingQueriesForTestQuery(sliceTargetSmall)
+        .optionSettingQueriesForBaseline(sliceTargetDefault)
+        .unOrdered()
+        .sqlQuery(query)
+        .sqlBaselineQuery(query)
+        .build()
+        .run();
+    } finally {
+      test(sliceTargetDefault);
+    }
+  }
+
+  @Test // DRILL-4147 // union-all above a hash join
+  public void testDrill4147_3() throws Exception {
+    final String l = FileUtils.getResourceAsFile("/multilevel/parquet/1994").toURI().toString();
+    final String r = FileUtils.getResourceAsFile("/multilevel/parquet/1995").toURI().toString();
+
+    final String query = String.format("SELECT o_custkey FROM \n" +
+        " (select o1.o_custkey from dfs_test.`%s` o1 inner join dfs_test.`%s` o2 on o1.o_orderkey
= o2.o_custkey) \n" +
+        " Union All SELECT o_custkey FROM dfs_test.`%s` where o_custkey < 10", l, r, l);
+
+    // Validate the plan
+    final String[] expectedPlan = {"(?s)UnionExchange.*UnionAll.*HashJoin.*"};
+    final String[] excludedPlan = {};
+
+    try {
+      test(sliceTargetSmall);
+      PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan);
+
+      testBuilder()
+        .optionSettingQueriesForTestQuery(sliceTargetSmall)
+        .optionSettingQueriesForBaseline(sliceTargetDefault)
+        .unOrdered()
+        .sqlQuery(query)
+        .sqlBaselineQuery(query)
+        .build()
+        .run();
+    } finally {
+      test(sliceTargetDefault);
+    }
+  }
+
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/drill/blob/0700c6b1/exec/java-exec/src/test/java/org/apache/drill/TestUnionDistinct.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestUnionDistinct.java b/exec/java-exec/src/test/java/org/apache/drill/TestUnionDistinct.java
index a615136..72a733a 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestUnionDistinct.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestUnionDistinct.java
@@ -18,6 +18,7 @@
 package org.apache.drill;
 
 import com.google.common.collect.Lists;
+
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.drill.common.exceptions.UserException;
 import org.apache.drill.common.expression.SchemaPath;
@@ -32,7 +33,10 @@ import java.util.List;
 public class TestUnionDistinct extends BaseTestQuery {
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestUnionDistinct.class);
 
-  @Test  // Simple Unionover two scans
+  private static final String sliceTargetSmall = "alter session set `planner.slice_target`
= 1";
+  private static final String sliceTargetDefault = "alter session reset `planner.slice_target`";
+
+  @Test  // Simple Union over two scans
   public void testUnionDistinct1() throws Exception {
     String query = "(select n_regionkey from cp.`tpch/nation.parquet`) union (select r_regionkey
from cp.`tpch/region.parquet`)";
 
@@ -821,4 +825,34 @@ public class TestUnionDistinct extends BaseTestQuery {
         .build()
         .run();
   }
+
+  @Test // DRILL-4147 // union-distinct base case
+  public void testDrill4147_1() throws Exception {
+    final String l = FileUtils.getResourceAsFile("/multilevel/parquet/1994").toURI().toString();
+    final String r = FileUtils.getResourceAsFile("/multilevel/parquet/1995").toURI().toString();
+
+    final String query = String.format("SELECT o_custkey FROM dfs_test.`%s` \n" +
+        "Union distinct SELECT o_custkey FROM dfs_test.`%s`", l, r);
+
+    // Validate the plan
+    final String[] expectedPlan = {"(?s)UnionExchange.*HashAgg.*HashToRandomExchange.*UnionAll.*"};
+    final String[] excludedPlan = {};
+
+    try {
+      test(sliceTargetSmall);
+      PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan);
+
+      testBuilder()
+        .optionSettingQueriesForTestQuery(sliceTargetSmall)
+        .optionSettingQueriesForBaseline(sliceTargetDefault)
+        .unOrdered()
+        .sqlQuery(query)
+        .sqlBaselineQuery(query)
+        .build()
+        .run();
+    } finally {
+      test(sliceTargetDefault);
+    }
+  }
+
 }


Mime
View raw message