drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jacq...@apache.org
Subject [3/3] drill git commit: DRILL-2897: Avoid parallelization when Drill is running limit 0 query.
Date Sat, 09 May 2015 15:51:49 GMT
DRILL-2897: Avoid parallelization when Drill is running limit 0 query.


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/9d209622
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/9d209622
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/9d209622

Branch: refs/heads/master
Commit: 9d209622934cb2be929a1591fd6a059165e1ac39
Parents: 960f876
Author: Jacques Nadeau <jacques@apache.org>
Authored: Sun Apr 26 22:35:57 2015 -0700
Committer: Jacques Nadeau <jacques@apache.org>
Committed: Sat May 9 08:03:59 2015 -0700

----------------------------------------------------------------------
 .../exec/planner/physical/PlannerSettings.java  |   7 +-
 .../planner/sql/handlers/DefaultSqlHandler.java |  57 +++----
 .../planner/sql/handlers/FindLimit0Visitor.java | 119 +++++++++++++++
 .../org/apache/drill/TestExampleQueries.java    |   1 +
 .../java/org/apache/drill/TestTpchLimit0.java   | 149 +++++++++++++++++++
 5 files changed, 305 insertions(+), 28 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/9d209622/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
index 7d8dd97..d4fb4f2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
@@ -33,6 +33,7 @@ public class PlannerSettings implements Context{
 
   private int numEndPoints = 0;
   private boolean useDefaultCosting = false; // True: use default Optiq costing, False: use
Drill costing
+  private boolean forceSingleMode;
 
   public static final int MAX_BROADCAST_THRESHOLD = Integer.MAX_VALUE;
   public static final int DEFAULT_IDENTIFIER_MAX_LENGTH = 1024;
@@ -81,7 +82,11 @@ public class PlannerSettings implements Context{
   }
 
   public boolean isSingleMode() {
-    return options.getOption(EXCHANGE.getOptionName()).bool_val;
+    return forceSingleMode || options.getOption(EXCHANGE.getOptionName()).bool_val;
+  }
+
+  public void forceSingleMode() {
+    forceSingleMode = true;
   }
 
   public int numEndPoints() {

http://git-wip-us.apache.org/repos/asf/drill/blob/9d209622/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
index eda1b5f..188aaa9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
@@ -22,19 +22,23 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 
-import org.apache.calcite.rel.core.Project;
-import org.apache.calcite.rel.logical.LogicalProject;
+import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.plan.hep.HepPlanner;
+import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.rel.rules.ProjectRemoveRule;
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rex.RexBuilder;
 import org.apache.calcite.rex.RexNode;
 import org.apache.calcite.rex.RexUtil;
+import org.apache.calcite.sql.SqlExplainLevel;
+import org.apache.calcite.sql.SqlNode;
 import org.apache.calcite.sql.TypedSqlNode;
 import org.apache.calcite.sql.validate.SqlValidatorUtil;
 import org.apache.calcite.tools.Planner;
 import org.apache.calcite.tools.RelConversionException;
 import org.apache.calcite.tools.ValidationException;
-
 import org.apache.drill.common.JSONOptions;
 import org.apache.drill.common.logical.PlanProperties;
 import org.apache.drill.common.logical.PlanProperties.Generator.ResultMode;
@@ -55,7 +59,6 @@ import org.apache.drill.exec.planner.physical.DrillDistributionTrait;
 import org.apache.drill.exec.planner.physical.PhysicalPlanCreator;
 import org.apache.drill.exec.planner.physical.PlannerSettings;
 import org.apache.drill.exec.planner.physical.Prel;
-import org.apache.drill.exec.planner.physical.ProjectPrel;
 import org.apache.drill.exec.planner.physical.explain.PrelSequencer;
 import org.apache.drill.exec.planner.physical.visitor.ComplexToJsonPrelVisitor;
 import org.apache.drill.exec.planner.physical.visitor.ExcessiveExchangeIdentifier;
@@ -77,13 +80,6 @@ import org.apache.drill.exec.util.Pointer;
 import org.apache.drill.exec.work.foreman.ForemanSetupException;
 import org.apache.drill.exec.work.foreman.SqlUnsupportedException;
 import org.apache.drill.exec.work.foreman.UnsupportedRelOperatorException;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.plan.RelOptPlanner;
-import org.apache.calcite.plan.RelOptUtil;
-import org.apache.calcite.plan.RelTraitSet;
-import org.apache.calcite.plan.hep.HepPlanner;
-import org.apache.calcite.sql.SqlExplainLevel;
-import org.apache.calcite.sql.SqlNode;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.google.common.base.Preconditions;
@@ -215,23 +211,23 @@ public class DefaultSqlHandler extends AbstractSqlHandler {
     return rel;
   }
 
-  protected RelNode preprocessNode(RelNode rel) throws SqlUnsupportedException{
-     /* Traverse the tree to do the following pre-processing tasks:
-     * 1. replace the convert_from, convert_to function to actual implementations
-     * Eg: convert_from(EXPR, 'JSON') be converted to convert_fromjson(EXPR);
-     * TODO: Ideally all function rewrites would move here instead of DrillOptiq
+  protected RelNode preprocessNode(RelNode rel) throws SqlUnsupportedException {
+    /*
+     * Traverse the tree to do the following pre-processing tasks: 1. replace the convert_from,
convert_to function to
+     * actual implementations Eg: convert_from(EXPR, 'JSON') be converted to convert_fromjson(EXPR);
TODO: Ideally all
+     * function rewrites would move here instead of DrillOptiq.
      *
-     * 2. see where the tree contains unsupported functions;
-     * throw SqlUnsupportedException if there is
+     * 2. see where the tree contains unsupported functions; throw SqlUnsupportedException
if there is any.
      */
 
-     PreProcessLogicalRel visitor = PreProcessLogicalRel.createVisitor(planner.getTypeFactory(),
context.getDrillOperatorTable());
-     try {
-        rel = rel.accept(visitor);
-     } catch(UnsupportedOperationException ex) {
-        visitor.convertException();
-       throw ex;
-     }
+    PreProcessLogicalRel visitor = PreProcessLogicalRel.createVisitor(planner.getTypeFactory(),
+        context.getDrillOperatorTable());
+    try {
+      rel = rel.accept(visitor);
+    } catch (UnsupportedOperationException ex) {
+      visitor.convertException();
+      throw ex;
+    }
 
     return rel;
   }
@@ -243,9 +239,16 @@ public class DefaultSqlHandler extends AbstractSqlHandler {
       if (convertedRelNode instanceof DrillStoreRel) {
         throw new UnsupportedOperationException();
       } else {
+
+        // If the query contains a limit 0 clause, disable distributed mode since it is overkill
for determining schema.
+        if (FindLimit0Visitor.containsLimit0(convertedRelNode)) {
+          context.getPlannerSettings().forceSingleMode();
+        }
+
         // Put a non-trivial topProject to ensure the final output field name is preserved,
when necessary.
-        DrillRel topPreservedNameProj = addRenamedProject((DrillRel)convertedRelNode, validatedRowType);
-        return new DrillScreenRel(topPreservedNameProj.getCluster(), topPreservedNameProj.getTraitSet(),
topPreservedNameProj);
+        DrillRel topPreservedNameProj = addRenamedProject((DrillRel) convertedRelNode, validatedRowType);
+        return new DrillScreenRel(topPreservedNameProj.getCluster(), topPreservedNameProj.getTraitSet(),
+            topPreservedNameProj);
       }
     } catch (RelOptPlanner.CannotPlanException ex) {
       logger.error(ex.getMessage());

http://git-wip-us.apache.org/repos/asf/drill/blob/9d209622/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/FindLimit0Visitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/FindLimit0Visitor.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/FindLimit0Visitor.java
new file mode 100644
index 0000000..d2c5fa6
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/FindLimit0Visitor.java
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.sql.handlers;
+
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.RelShuttleImpl;
+import org.apache.calcite.rel.logical.LogicalAggregate;
+import org.apache.calcite.rel.logical.LogicalIntersect;
+import org.apache.calcite.rel.logical.LogicalJoin;
+import org.apache.calcite.rel.logical.LogicalMinus;
+import org.apache.calcite.rel.logical.LogicalSort;
+import org.apache.calcite.rel.logical.LogicalUnion;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.drill.exec.planner.logical.DrillLimitRel;
+
+/**
+ * Visitor that will identify whether the root portion of the RelNode tree contains a limit
0 pattern. In this case, we
+ * inform the planner settings that this plan should be run as a single node plan to reduce
the overhead associated with
+ * executing a schema-only query.
+ */
+public class FindLimit0Visitor extends RelShuttleImpl {
+  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FindLimit0Visitor.class);
+
+  private boolean contains = false;
+
+  public static boolean containsLimit0(RelNode rel) {
+    FindLimit0Visitor visitor = new FindLimit0Visitor();
+    rel.accept(visitor);
+    return visitor.isContains();
+  }
+
+  private FindLimit0Visitor() {
+  }
+
+  boolean isContains() {
+    return contains;
+  }
+
+  private boolean isLimit0(RexNode fetch) {
+    if (fetch != null && fetch.isA(SqlKind.LITERAL)) {
+      RexLiteral l = (RexLiteral) fetch;
+      switch (l.getTypeName()) {
+      case BIGINT:
+      case INTEGER:
+      case DECIMAL:
+        if (((long) l.getValue2()) == 0) {
+          return true;
+        }
+      }
+    }
+    return false;
+  }
+
+  @Override
+  public RelNode visit(LogicalSort sort) {
+    if (isLimit0(sort.fetch)) {
+      contains = true;
+      return sort;
+    }
+
+    return super.visit(sort);
+  }
+
+  @Override
+  public RelNode visit(RelNode other) {
+    if (other instanceof DrillLimitRel) {
+      if (isLimit0(((DrillLimitRel) other).getFetch())) {
+        contains = true;
+        return other;
+      }
+    }
+
+    return super.visit(other);
+  }
+
+  // The following set of RelNodes should terminate a search for the limit 0 pattern as they
want convey its meaning.
+
+  @Override
+  public RelNode visit(LogicalAggregate aggregate) {
+    return aggregate;
+  }
+
+  @Override
+  public RelNode visit(LogicalIntersect intersect) {
+    return intersect;
+  }
+
+  @Override
+  public RelNode visit(LogicalJoin join) {
+    return join;
+  }
+
+  @Override
+  public RelNode visit(LogicalMinus minus) {
+    return minus;
+  }
+
+  @Override
+  public RelNode visit(LogicalUnion union) {
+    return union;
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/9d209622/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
index 7e07500..18336cf 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
@@ -284,6 +284,7 @@ public class TestExampleQueries extends BaseTestQuery{
   }
 
   @Test
+  @Ignore("DRILL-3004")
   public void testJoin() throws Exception{
     test("alter session set `planner.enable_hashjoin` = false");
     test("SELECT\n" +

http://git-wip-us.apache.org/repos/asf/drill/blob/9d209622/exec/java-exec/src/test/java/org/apache/drill/TestTpchLimit0.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestTpchLimit0.java b/exec/java-exec/src/test/java/org/apache/drill/TestTpchLimit0.java
new file mode 100644
index 0000000..22471c8
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestTpchLimit0.java
@@ -0,0 +1,149 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class TestTpchLimit0 extends BaseTestQuery{
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestTpchLimit0.class);
+
+  private void testLimitZero(String fileName) throws Exception {
+    String query = getFile(fileName);
+    query = "ALTER SESSION SET `planner.slice_target` = 1; select * from \n(" + query.replace(";",
")xyz limit 0;");
+    test(query);
+  }
+
+  @Test
+  public void tpch01() throws Exception{
+    testLimitZero("queries/tpch/01.sql");
+  }
+
+  @Test
+  @Ignore // DRILL-512
+  public void tpch02() throws Exception{
+    testLimitZero("queries/tpch/02.sql");
+  }
+
+  @Test
+  public void tpch03() throws Exception{
+    testLimitZero("queries/tpch/03.sql");
+  }
+
+  @Test
+  public void tpch04() throws Exception{
+    testLimitZero("queries/tpch/04.sql");
+  }
+
+  @Test
+  public void tpch05() throws Exception{
+    testLimitZero("queries/tpch/05.sql");
+  }
+
+  @Test
+  public void tpch06() throws Exception{
+    testLimitZero("queries/tpch/06.sql");
+  }
+
+  @Test
+  public void tpch07() throws Exception{
+    testLimitZero("queries/tpch/07.sql");
+  }
+
+  @Test
+  public void tpch08() throws Exception{
+    testLimitZero("queries/tpch/08.sql");
+  }
+
+  @Test
+  public void tpch09() throws Exception{
+    testLimitZero("queries/tpch/09.sql");
+  }
+
+  @Test
+  public void tpch10() throws Exception{
+    testLimitZero("queries/tpch/10.sql");
+  }
+
+  @Test
+  @Ignore // Cartesian problem
+  public void tpch11() throws Exception{
+    testLimitZero("queries/tpch/11.sql");
+  }
+
+  @Test
+  public void tpch12() throws Exception{
+    testLimitZero("queries/tpch/12.sql");
+  }
+
+  @Test
+  public void tpch13() throws Exception{
+    testLimitZero("queries/tpch/13.sql");
+  }
+
+  @Test
+  public void tpch14() throws Exception{
+    testLimitZero("queries/tpch/14.sql");
+  }
+
+  @Test
+  @Ignore //
+  public void tpch15() throws Exception{
+    testLimitZero("queries/tpch/15.sql");
+  }
+
+  @Test
+  @Ignore // invalid plan, due to Nulls value NOT IN sub-q
+  public void tpch16() throws Exception{
+    testLimitZero("queries/tpch/16.sql");
+  }
+
+  @Test
+  @Ignore //
+  public void tpch17() throws Exception{
+    testLimitZero("queries/tpch/17.sql");
+  }
+
+  @Test
+  public void tpch18() throws Exception{
+    testLimitZero("queries/tpch/18.sql");
+  }
+
+  @Test
+  @Ignore // DRILL-519
+  public void tpch19() throws Exception{
+    testLimitZero("queries/tpch/19.sql");
+  }
+
+  @Test
+  public void tpch20() throws Exception{
+    testLimitZero("queries/tpch/20.sql");
+  }
+
+  @Test
+  @Ignore
+  public void tpch21() throws Exception{
+    testLimitZero("queries/tpch/21.sql");
+  }
+
+  @Test
+  @Ignore // DRILL-518
+  public void tpch22() throws Exception{
+    testLimitZero("queries/tpch/22.sql");
+  }
+}


Mime
View raw message