drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j..@apache.org
Subject drill git commit: DRILL-2069: Fix star query prefix logic it's used with subquery in where clause. Convert some star query query unit test to verify query results.
Date Tue, 03 Feb 2015 22:26:37 GMT
Repository: drill
Updated Branches:
  refs/heads/master 09f7fb202 -> fd005d276


DRILL-2069:  Fix star query prefix logic it's used with subquery in where clause. Convert
some star query query unit test to verify query results.

Add new unit test case based on review comments.


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/fd005d27
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/fd005d27
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/fd005d27

Branch: refs/heads/master
Commit: fd005d2761461f0c50994c0e4fb37c7c50f2370b
Parents: 09f7fb2
Author: Jinfeng Ni <jni@maprtech.com>
Authored: Mon Jan 26 11:22:50 2015 -0800
Committer: Jinfeng Ni <jni@maprtech.com>
Committed: Mon Feb 2 11:07:22 2015 -0800

----------------------------------------------------------------------
 .../drill/exec/planner/StarColumnHelper.java    |  22 ++-
 .../physical/visitor/StarColumnConverter.java   |  65 +++++---
 .../java/org/apache/drill/TestStarQueries.java  | 153 +++++++++++++++++--
 3 files changed, 201 insertions(+), 39 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/fd005d27/exec/java-exec/src/main/java/org/apache/drill/exec/planner/StarColumnHelper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/StarColumnHelper.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/StarColumnHelper.java
index b9dfda3..d288479 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/StarColumnHelper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/StarColumnHelper.java
@@ -23,6 +23,8 @@ import java.util.Map;
 import java.util.Set;
 
 import org.eigenbase.reltype.RelDataType;
+import org.eigenbase.rex.RexInputRef;
+import org.eigenbase.rex.RexNode;
 
 public class StarColumnHelper {
 
@@ -48,12 +50,30 @@ public class StarColumnHelper {
     return false;
   }
 
+  public static boolean containsStarColumnInProject(RelDataType inputRowType, List<RexNode>
projExprs) {
+    if (! inputRowType.isStruct()) {
+      return false;
+    }
+
+    for (RexNode expr : projExprs) {
+      if (expr instanceof RexInputRef) {
+        String name = inputRowType.getFieldNames().get(((RexInputRef) expr).getIndex());
+
+        if (name.startsWith(STAR_COLUMN)) {
+          return true;
+        }
+      }
+    }
+
+    return false;
+  }
+
   public static boolean isPrefixedStarColumn(String fieldName) {
     return fieldName.indexOf(PREFIXED_STAR_COLUMN) > 0 ; // the delimiter * starts at
none-zero position.
   }
 
   public static boolean isNonPrefixedStarColumn(String fieldName) {
-    return fieldName.startsWith("*");
+    return fieldName.startsWith(STAR_COLUMN);
   }
 
   public static boolean isStarColumn(String fieldName) {

http://git-wip-us.apache.org/repos/asf/drill/blob/fd005d27/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/StarColumnConverter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/StarColumnConverter.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/StarColumnConverter.java
index fcaafc8..edf1a2e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/StarColumnConverter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/StarColumnConverter.java
@@ -32,6 +32,7 @@ import org.apache.drill.exec.planner.physical.ScanPrel;
 import org.apache.drill.exec.planner.physical.ScreenPrel;
 import org.apache.drill.exec.planner.physical.WriterPrel;
 import org.eigenbase.rel.RelNode;
+import org.eigenbase.rel.rules.RemoveTrivialProjectRule;
 import org.eigenbase.reltype.RelDataType;
 import org.eigenbase.reltype.RelDataTypeField;
 import org.eigenbase.rex.RexInputRef;
@@ -113,52 +114,68 @@ public class StarColumnConverter extends BasePrelVisitor<Prel, boolean[],
Runtim
   }
 
   @Override
-  public Prel visitPrel(Prel prel, boolean [] prefixedForStar) throws RuntimeException {
+  public Prel visitProject(ProjectPrel prel, boolean[] prefixedForStar) throws RuntimeException
{
+    ProjectPrel proj = (ProjectPrel) prel;
+
     // Require prefix rename : there exists other expression, in addition to a star column.
     if (!prefixedForStar[0]  // not set yet.
-        && StarColumnHelper.containsStarColumn(prel.getRowType())
+        && StarColumnHelper.containsStarColumnInProject(prel.getChild().getRowType(),
proj.getProjects())
         && prel.getRowType().getFieldNames().size() > 1) {
       prefixedForStar[0] = true;
     }
 
-    List<RelNode> children = Lists.newArrayList();
-    for (Prel child : prel) {
-      child = child.accept(this, prefixedForStar);
-      children.add(child);
-    }
-
     // For project, we need make sure that the project's field name is same as the input,
     // when the project expression is RexInPutRef, since we may insert a PAS which will
     // rename the projected fields.
-    if (prel instanceof ProjectPrel) {
 
-      ProjectPrel proj = (ProjectPrel) prel;
 
-      RelNode child = children.get(0);
 
-      List<String> fieldNames = Lists.newArrayList();
+    RelNode child = ((Prel) prel.getInput(0)).accept(INSTANCE, prefixedForStar);
 
-      for (Pair<String, RexNode> pair : Pair.zip(prel.getRowType().getFieldNames(),
proj.getProjects())) {
-        if (pair.right instanceof RexInputRef) {
-          String name = child.getRowType().getFieldNames().get(((RexInputRef) pair.right).getIndex());
-          fieldNames.add(name);
-        } else {
-          fieldNames.add(pair.left);
-        }
+    List<String> fieldNames = Lists.newArrayList();
+
+    for (Pair<String, RexNode> pair : Pair.zip(prel.getRowType().getFieldNames(), proj.getProjects()))
{
+      if (pair.right instanceof RexInputRef) {
+        String name = child.getRowType().getFieldNames().get(((RexInputRef) pair.right).getIndex());
+        fieldNames.add(name);
+      } else {
+        fieldNames.add(pair.left);
       }
+    }
 
-      // Make sure the field names are unique : no allow of duplicate field names in a rowType.
-      fieldNames = makeUniqueNames(fieldNames);
+    // Make sure the field names are unique : no allow of duplicate field names in a rowType.
+    fieldNames = makeUniqueNames(fieldNames);
 
-      RelDataType rowType = RexUtil.createStructType(prel.getCluster().getTypeFactory(),
proj.getProjects(), fieldNames);
+    RelDataType rowType = RexUtil.createStructType(prel.getCluster().getTypeFactory(), proj.getProjects(),
fieldNames);
 
-      return (Prel) proj.copy(proj.getTraitSet(),children.get(0), proj.getProjects(), rowType);
+    ProjectPrel newProj = (ProjectPrel) proj.copy(proj.getTraitSet(), child, proj.getProjects(),
rowType);
+
+    if (RemoveTrivialProjectRule.isTrivial(newProj)) {
+      return (Prel) child;
     } else {
-      return (Prel) prel.copy(prel.getTraitSet(), children);
+      return newProj;
     }
   }
 
   @Override
+  public Prel visitPrel(Prel prel, boolean [] prefixedForStar) throws RuntimeException {
+    // Require prefix rename : there exists other expression, in addition to a star column.
+    if (!prefixedForStar[0]  // not set yet.
+        && StarColumnHelper.containsStarColumn(prel.getRowType())
+        && prel.getRowType().getFieldNames().size() > 1) {
+      prefixedForStar[0] = true;
+    }
+
+    List<RelNode> children = Lists.newArrayList();
+    for (Prel child : prel) {
+      child = child.accept(this, prefixedForStar);
+      children.add(child);
+    }
+
+    return (Prel) prel.copy(prel.getTraitSet(), children);
+  }
+
+  @Override
   public Prel visitScan(ScanPrel scanPrel, boolean [] prefixedForStar) throws RuntimeException
{
     if (StarColumnHelper.containsStarColumn(scanPrel.getRowType()) && prefixedForStar[0]
) {
 

http://git-wip-us.apache.org/repos/asf/drill/blob/fd005d27/exec/java-exec/src/test/java/org/apache/drill/TestStarQueries.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestStarQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestStarQueries.java
index 3d7c31e..5447fa6 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestStarQueries.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestStarQueries.java
@@ -119,52 +119,116 @@ public class TestStarQueries extends BaseTestQuery{
 
   @Test
   public void testSelStarOrderBy() throws Exception{
-    test("select * from cp.`employee.json` order by last_name");
+    testBuilder()
+        .ordered()
+        .sqlQuery(" select * from cp.`employee.json` order by last_name")
+        .sqlBaselineQuery(" select employee_id, full_name,first_name,last_name,position_id,position_title,store_id,"
+
+            " department_id,birth_date,hire_date,salary,supervisor_id,education_level,marital_status,gender,management_role
" +
+            " from cp.`employee.json` " +
+            " order by last_name ")
+        .build().run();
+
   }
 
   @Test
   public void testSelStarOrderByLimit() throws Exception{
-    test("select * from cp.`employee.json` order by last_name limit 2;");
+    testBuilder()
+        .ordered()
+        .sqlQuery(" select * from cp.`employee.json` order by last_name limit 2")
+        .sqlBaselineQuery(" select employee_id, full_name,first_name,last_name,position_id,position_title,store_id,"
+
+            " department_id,birth_date,hire_date,salary,supervisor_id,education_level,marital_status,gender,management_role
" +
+            " from cp.`employee.json` " +
+            " order by last_name limit 2")
+        .build().run();
+
   }
 
   @Test
   public void testSelStarPlusRegCol() throws Exception{
-    test("select *, n_nationkey from cp.`tpch/nation.parquet` limit 2;");
+    testBuilder()
+        .unOrdered()
+        .sqlQuery("select *, n_nationkey as key2 from cp.`tpch/nation.parquet` order by n_name
limit 2")
+        .sqlBaselineQuery("select n_comment, n_name, n_nationkey, n_regionkey, n_nationkey
as key2 from cp.`tpch/nation.parquet` order by n_name limit 2")
+        .build().run();
+
   }
 
   @Test
   public void testSelStarWhereOrderBy() throws Exception{
-    test("select * from cp.`employee.json` where first_name = 'James' order by last_name");
+    testBuilder()
+        .ordered()
+        .sqlQuery("select * from cp.`employee.json` where first_name = 'James' order by last_name")
+        .sqlBaselineQuery("select employee_id, full_name,first_name,last_name,position_id,position_title,store_id,"
+
+            " department_id,birth_date,hire_date,salary,supervisor_id,education_level,marital_status,gender,management_role
" +
+            " from cp.`employee.json` " +
+            " where first_name = 'James' order by last_name")
+        .build().run();
+
   }
 
   @Test
   public void testSelStarJoin() throws Exception {
-    test("select * from cp.`tpch/nation.parquet` n, cp.`tpch/region.parquet` r where n.n_regionkey
= r.r_regionkey order by n.n_name;");
+    testBuilder()
+        .ordered()
+        .sqlQuery("select * from cp.`tpch/nation.parquet` n, cp.`tpch/region.parquet` r where
n.n_regionkey = r.r_regionkey order by n.n_name")
+        .sqlBaselineQuery("select n.n_nationkey, n.n_name,n.n_regionkey,n.n_comment,r.r_regionkey,r.r_name,
r.r_comment from cp.`tpch/nation.parquet` n, cp.`tpch/region.parquet` r where n.n_regionkey
= r.r_regionkey order by n.n_name")
+        .build().run();
+
   }
 
   @Test
   public void testSelLeftStarJoin() throws Exception {
-    test("select n.* from cp.`tpch/nation.parquet` n, cp.`tpch/region.parquet` r where n.n_regionkey
= r.r_regionkey order by n.n_name;");
+    testBuilder()
+        .ordered()
+        .sqlQuery("select n.* from cp.`tpch/nation.parquet` n, cp.`tpch/region.parquet` r
where n.n_regionkey = r.r_regionkey order by n.n_name")
+        .sqlBaselineQuery("select n.n_nationkey, n.n_name, n.n_regionkey, n.n_comment from
cp.`tpch/nation.parquet` n, cp.`tpch/region.parquet` r where n.n_regionkey = r.r_regionkey
order by n.n_name")
+        .build().run();
+
   }
 
   @Test
   public void testSelRightStarJoin() throws Exception {
-    test("select r.* from cp.`tpch/nation.parquet` n, cp.`tpch/region.parquet` r where n.n_regionkey
= r.r_regionkey order by n.n_name;");
+    testBuilder()
+        .ordered()
+        .sqlQuery("select r.* from cp.`tpch/nation.parquet` n, cp.`tpch/region.parquet` r
where n.n_regionkey = r.r_regionkey order by n.n_name")
+        .sqlBaselineQuery("select r.r_regionkey, r.r_name, r.r_comment from cp.`tpch/nation.parquet`
n, cp.`tpch/region.parquet` r where n.n_regionkey = r.r_regionkey order by n.n_name")
+        .build().run();
+
   }
 
   @Test
   public void testSelStarRegColConstJoin() throws Exception {
-    test("select *, n.n_nationkey as n_nationkey0, 1 + 2 as constant from cp.`tpch/nation.parquet`
n, cp.`tpch/region.parquet` r where n.n_regionkey = r.r_regionkey order by n.n_name;");
+    testBuilder()
+        .ordered()
+        .sqlQuery("select *, n.n_nationkey as n_nationkey0, 1 + 2 as constant from cp.`tpch/nation.parquet`
n, cp.`tpch/region.parquet` r where n.n_regionkey = r.r_regionkey order by n.n_name")
+        .sqlBaselineQuery(" select n.n_nationkey, n.n_name, n.n_regionkey, n.n_comment, r.r_regionkey,
r.r_name, r.r_comment, " +
+            " n.n_nationkey as n_nationkey0, 1 + 2 as constant " +
+            " from cp.`tpch/nation.parquet` n, cp.`tpch/region.parquet` r " +
+            " where n.n_regionkey = r.r_regionkey " +
+            " order by n.n_name")
+        .build().run();
+
   }
 
   @Test
   public void testSelStarBothSideJoin() throws Exception {
-    test("select n.*, r.* from cp.`tpch/nation.parquet` n, cp.`tpch/region.parquet` r where
n.n_regionkey = r.r_regionkey;");
+    testBuilder()
+        .unOrdered()
+        .sqlQuery("select n.*, r.* from cp.`tpch/nation.parquet` n, cp.`tpch/region.parquet`
r where n.n_regionkey = r.r_regionkey")
+        .sqlBaselineQuery("select n.n_nationkey,n.n_name,n.n_regionkey,n.n_comment,r.r_regionkey,r.r_name,r.r_comment
from cp.`tpch/nation.parquet` n, cp.`tpch/region.parquet` r where n.n_regionkey = r.r_regionkey
order by n.n_name")
+        .build().run();
+
   }
 
   @Test
   public void testSelStarJoinSameColName() throws Exception {
-    test("select * from cp.`tpch/nation.parquet` n1, cp.`tpch/nation.parquet` n2 where n1.n_nationkey
= n2.n_nationkey;");
+    testBuilder()
+        .unOrdered()
+        .sqlQuery("select * from cp.`tpch/nation.parquet` n1, cp.`tpch/nation.parquet` n2
where n1.n_nationkey = n2.n_nationkey")
+        .sqlBaselineQuery("select n1.n_nationkey,n1.n_name,n1.n_regionkey,n1.n_comment,n2.n_nationkey,n2.n_name,n2.n_regionkey,
n2.n_comment " +
+            "from cp.`tpch/nation.parquet` n1, cp.`tpch/nation.parquet` n2 where n1.n_nationkey
= n2.n_nationkey")
+        .build().run();
+
   }
 
   @Test // DRILL-1293
@@ -304,10 +368,23 @@ public class TestStarQueries extends BaseTestQuery{
 
   @Test  // DRILL-1889
   public void testStarWithOtherExpression() throws Exception {
-    test("select *  from cp.`tpch/nation.parquet` order by substr(n_name, 2, 5) limit 3");
-    test("select *, n_nationkey + 5 from cp.`tpch/nation.parquet` limit 3");
-    test("select *  from cp.`tpch/nation.parquet` where n_nationkey + 5 > 10 limit 3");
-    test("select * from cp.`tpch/nation.parquet` order by random()");
+    testBuilder()
+        .ordered()
+        .sqlQuery("select *  from cp.`tpch/nation.parquet` order by substr(n_name, 2, 5)
limit 3")
+        .sqlBaselineQuery("select n_comment, n_name, n_nationkey, n_regionkey from cp.`tpch/nation.parquet`
order by substr(n_name, 2, 5) limit 3 ")
+        .build().run();
+
+    testBuilder()
+        .ordered()
+        .sqlQuery("select *, n_nationkey + 5 as myexpr from cp.`tpch/nation.parquet` limit
3")
+        .sqlBaselineQuery("select n_comment, n_name, n_nationkey, n_regionkey, n_nationkey
+ 5 as myexpr from cp.`tpch/nation.parquet` order by n_nationkey limit 3")
+        .build().run();
+
+    testBuilder()
+        .ordered()
+        .sqlQuery("select *  from cp.`tpch/nation.parquet` where n_nationkey + 5 > 10
limit 3")
+        .sqlBaselineQuery("select n_comment, n_name, n_nationkey, n_regionkey  from cp.`tpch/nation.parquet`
where n_nationkey + 5 > 10 order by n_nationkey limit 3")
+        .build().run();
   }
 
   @Test // DRILL-1500
@@ -323,4 +400,52 @@ public class TestStarQueries extends BaseTestQuery{
     .build().run();
   }
 
+  @Test // DRILL-2069
+  public void testStarInSubquery() throws Exception {
+    testBuilder()
+        .unOrdered()
+        .sqlQuery("select * from cp.`tpch/nation.parquet` where n_regionkey in (select r_regionkey
from cp.`tpch/region.parquet`)")
+        .sqlBaselineQuery("select n_nationkey, n_name, n_regionkey, n_comment from cp.`tpch/nation.parquet`
where n_regionkey in (select r_regionkey from cp.`tpch/region.parquet`)")
+        .build().run();
+
+    // multiple columns in "IN" subquery predicates.
+    testBuilder()
+        .unOrdered()
+        .sqlQuery("select * from cp.`tpch/nation.parquet` where (n_nationkey, n_name) in
( select n_nationkey, n_name from cp.`tpch/nation.parquet`)")
+        .sqlBaselineQuery("select n_nationkey, n_name, n_regionkey, n_comment from cp.`tpch/nation.parquet`
where (n_nationkey, n_name) in ( select n_nationkey, n_name from cp.`tpch/nation.parquet`)")
+        .build().run();
+
+    // Multiple in subquery predicates.
+    testBuilder()
+        .unOrdered()
+        .sqlQuery(
+            "select * from cp.`tpch/nation.parquet` " +
+            "where n_regionkey in ( select r_regionkey from cp.`tpch/region.parquet`) and
" +
+            "      n_name in (select n_name from cp.`tpch/nation.parquet`)")
+        .sqlBaselineQuery("select n_nationkey, n_name, n_regionkey, n_comment from cp.`tpch/nation.parquet`
" +
+            "where n_regionkey in ( select r_regionkey from cp.`tpch/region.parquet`) and
" +
+            "      n_name in (select n_name from cp.`tpch/nation.parquet`)")
+        .build().run();
+
+
+    // Both the out QB and SUBQ are join.
+    testBuilder()
+        .unOrdered()
+        .sqlQuery(
+            "select * from cp.`tpch/nation.parquet` n, cp.`tpch/region.parquet` r " +
+            "where n.n_regionkey = r.r_regionkey and " +
+            "       (n.n_nationkey, n.n_name) in " +
+            "          ( select n2.n_nationkey, n2.n_name " +
+            "            from cp.`tpch/nation.parquet` n2, cp.`tpch/region.parquet` r2 "
+
+            "            where n2.n_regionkey = r2.r_regionkey)")
+        .sqlBaselineQuery(
+            "select n.n_nationkey, n.n_name, n.n_regionkey, n.n_comment, r.r_regionkey, r.r_name,
r.r_comment " +
+            "from cp.`tpch/nation.parquet` n, cp.`tpch/region.parquet` r " +
+            "where n.n_regionkey = r.r_regionkey and " +
+            "       (n.n_nationkey, n.n_name) in " +
+            "          ( select n2.n_nationkey, n2.n_name " +
+            "            from cp.`tpch/nation.parquet` n2, cp.`tpch/region.parquet` r2 "
+
+            "            where n2.n_regionkey = r2.r_regionkey)")
+        .build().run();
+  }
 }


Mime
View raw message