drill-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Sudheesh Katkam <sudhe...@apache.org>
Subject Re: drill git commit: DRILL-3623: For limit 0 queries, optionally use a shorter execution path when result column types are known
Date Wed, 23 Mar 2016 05:39:41 GMT
Thanks!

Hsuan did most of the work; I am glad this is merged :)

On Tue, Mar 22, 2016 at 4:51 PM, Jacques Nadeau <jacques@apache.org> wrote:

> Awesome job on this Sudheesh.  Thanks for all the hard work. Thanks also to
> Sean for all his work on the previous patch.
> ---------- Forwarded message ----------
> From: <sudheesh@apache.org>
> Date: Mar 22, 2016 4:33 PM
> Subject: drill git commit: DRILL-3623: For limit 0 queries, optionally use
> a shorter execution path when result column types are known
> To: <commits@drill.apache.org>
> Cc:
>
> Repository: drill
> Updated Branches:
>   refs/heads/master 600ba9ee1 -> 5dbaafbe6
>
>
> DRILL-3623: For limit 0 queries, optionally use a shorter execution path
> when result column types are known
>
> + "planner.enable_limit0_optimization" option is disabled by default
>
> + Print plan in PlanTestBase if TEST_QUERY_PRINTING_SILENT is set
> + Fix DrillTestWrapper to verify expected and actual schema
> + Correct the schema of results in TestInbuiltHiveUDFs#testXpath_Double
>
> This closes #405
>
>
> Project: http://git-wip-us.apache.org/repos/asf/drill/repo
> Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/5dbaafbe
> Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/5dbaafbe
> Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/5dbaafbe
>
> Branch: refs/heads/master
> Commit: 5dbaafbe6651b0a284fef69d5c952d82ce506e20
> Parents: 600ba9e
> Author: Sudheesh Katkam <skatkam@maprtech.com>
> Authored: Tue Mar 22 15:21:51 2016 -0700
> Committer: Sudheesh Katkam <skatkam@maprtech.com>
> Committed: Tue Mar 22 16:19:01 2016 -0700
>
> ----------------------------------------------------------------------
>  .../drill/exec/fn/hive/TestInbuiltHiveUDFs.java |   2 +-
>  .../org/apache/drill/exec/ExecConstants.java    |   3 +
>  .../drill/exec/physical/base/ScanStats.java     |   6 +-
>  .../apache/drill/exec/planner/PlannerPhase.java |   2 +
>  .../planner/logical/DrillDirectScanRel.java     |  70 ++
>  .../exec/planner/physical/DirectScanPrule.java  |  49 ++
>  .../planner/sql/handlers/DefaultSqlHandler.java |  12 +
>  .../planner/sql/handlers/FindLimit0Visitor.java | 124 +++-
>  .../server/options/SystemOptionManager.java     |   1 +
>  .../exec/store/direct/DirectGroupScan.java      |  27 +-
>  .../java/org/apache/drill/DrillTestWrapper.java |  25 +-
>  .../java/org/apache/drill/PlanTestBase.java     |   9 +-
>  .../impl/limit/TestEarlyLimit0Optimization.java | 663 +++++++++++++++++++
>  13 files changed, 963 insertions(+), 30 deletions(-)
> ----------------------------------------------------------------------
>
>
>
> http://git-wip-us.apache.org/repos/asf/drill/blob/5dbaafbe/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestInbuiltHiveUDFs.java
> ----------------------------------------------------------------------
> diff --git
>
> a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestInbuiltHiveUDFs.java
>
> b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestInbuiltHiveUDFs.java
> index a287c89..a126aaa 100644
> ---
>
> a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestInbuiltHiveUDFs.java
> +++
>
> b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestInbuiltHiveUDFs.java
> @@ -58,7 +58,7 @@ public class TestInbuiltHiveUDFs extends HiveTestBase {
>
>      final TypeProtos.MajorType majorType =
> TypeProtos.MajorType.newBuilder()
>          .setMinorType(TypeProtos.MinorType.FLOAT8)
> -        .setMode(TypeProtos.DataMode.REQUIRED)
> +        .setMode(TypeProtos.DataMode.OPTIONAL)
>          .build();
>
>      final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema =
> Lists.newArrayList();
>
>
> http://git-wip-us.apache.org/repos/asf/drill/blob/5dbaafbe/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
> ----------------------------------------------------------------------
> diff --git
> a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
> b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
> index b8f25ad..963934d 100644
> --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
> +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
> @@ -202,6 +202,9 @@ public interface ExecConstants {
>    String AFFINITY_FACTOR_KEY = "planner.affinity_factor";
>    OptionValidator AFFINITY_FACTOR = new
> DoubleValidator(AFFINITY_FACTOR_KEY, 1.2d);
>
> +  String EARLY_LIMIT0_OPT_KEY = "planner.enable_limit0_optimization";
> +  BooleanValidator EARLY_LIMIT0_OPT = new
> BooleanValidator(EARLY_LIMIT0_OPT_KEY, false);
> +
>    String ENABLE_MEMORY_ESTIMATION_KEY =
> "planner.memory.enable_memory_estimation";
>    OptionValidator ENABLE_MEMORY_ESTIMATION = new
> BooleanValidator(ENABLE_MEMORY_ESTIMATION_KEY, false);
>
>
>
> http://git-wip-us.apache.org/repos/asf/drill/blob/5dbaafbe/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/ScanStats.java
> ----------------------------------------------------------------------
> diff --git
>
> a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/ScanStats.java
>
> b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/ScanStats.java
> index ba36931..1886c14 100644
> ---
>
> a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/ScanStats.java
> +++
>
> b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/ScanStats.java
> @@ -17,13 +17,13 @@
>   */
>  package org.apache.drill.exec.physical.base;
>
> -
>  public class ScanStats {
> -  static final org.slf4j.Logger logger =
> org.slf4j.LoggerFactory.getLogger(ScanStats.class);
> -
> +//  private static final org.slf4j.Logger logger =
> org.slf4j.LoggerFactory.getLogger(ScanStats.class);
>
>    public static final ScanStats TRIVIAL_TABLE = new
> ScanStats(GroupScanProperty.NO_EXACT_ROW_COUNT, 20, 1, 1);
>
> +  public static final ScanStats ZERO_RECORD_TABLE = new
> ScanStats(GroupScanProperty.EXACT_ROW_COUNT, 0, 1, 1);
> +
>    private final long recordCount;
>    private final float cpuCost;
>    private final float diskCost;
>
>
> http://git-wip-us.apache.org/repos/asf/drill/blob/5dbaafbe/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java
> ----------------------------------------------------------------------
> diff --git
>
> a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java
>
> b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java
> index 7ab7faf..57f2984 100644
> ---
>
> a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java
> +++
>
> b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java
> @@ -65,6 +65,7 @@ import
> org.apache.drill.exec.planner.logical.DrillWindowRule;
>  import
> org.apache.drill.exec.planner.logical.partition.ParquetPruneScanRule;
>  import org.apache.drill.exec.planner.logical.partition.PruneScanRule;
>  import org.apache.drill.exec.planner.physical.ConvertCountToDirectScan;
> +import org.apache.drill.exec.planner.physical.DirectScanPrule;
>  import org.apache.drill.exec.planner.physical.FilterPrule;
>  import org.apache.drill.exec.planner.physical.HashAggPrule;
>  import org.apache.drill.exec.planner.physical.HashJoinPrule;
> @@ -391,6 +392,7 @@ public enum PlannerPhase {
>      ruleList.add(LimitUnionExchangeTransposeRule.INSTANCE);
>      ruleList.add(UnionAllPrule.INSTANCE);
>      ruleList.add(ValuesPrule.INSTANCE);
> +    ruleList.add(DirectScanPrule.INSTANCE);
>
>      if (ps.isHashAggEnabled()) {
>        ruleList.add(HashAggPrule.INSTANCE);
>
>
> http://git-wip-us.apache.org/repos/asf/drill/blob/5dbaafbe/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillDirectScanRel.java
> ----------------------------------------------------------------------
> diff --git
>
> a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillDirectScanRel.java
>
> b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillDirectScanRel.java
> new file mode 100644
> index 0000000..013016a
> --- /dev/null
> +++
>
> b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillDirectScanRel.java
> @@ -0,0 +1,70 @@
> +/**
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements.  See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership.  The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License.  You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +package org.apache.drill.exec.planner.logical;
> +
> +import org.apache.calcite.plan.RelOptCluster;
> +import org.apache.calcite.plan.RelTraitSet;
> +import org.apache.calcite.rel.AbstractRelNode;
> +import org.apache.calcite.rel.RelWriter;
> +import org.apache.calcite.rel.type.RelDataType;
> +import org.apache.drill.common.logical.data.LogicalOperator;
> +import org.apache.drill.exec.planner.physical.PlannerSettings;
> +import org.apache.drill.exec.planner.physical.PrelUtil;
> +import org.apache.drill.exec.store.direct.DirectGroupScan;
> +
> +/**
> + * Logical RelNode representing a {@link DirectGroupScan}. This is not
> backed by a {@link DrillTable},
> + * unlike {@link DrillScanRel}.
> + */
> +public class DrillDirectScanRel extends AbstractRelNode implements
> DrillRel {
> +
> +  private final DirectGroupScan groupScan;
> +  private final RelDataType rowType;
> +
> +  public DrillDirectScanRel(RelOptCluster cluster, RelTraitSet traitSet,
> DirectGroupScan directGroupScan,
> +                            RelDataType rowType) {
> +    super(cluster, traitSet);
> +    this.groupScan = directGroupScan;
> +    this.rowType = rowType;
> +  }
> +
> +  @Override
> +  public LogicalOperator implement(DrillImplementor implementor) {
> +    return null;
> +  }
> +
> +  @Override
> +  public RelDataType deriveRowType() {
> +    return this.rowType;
> +  }
> +
> +  @Override
> +  public RelWriter explainTerms(RelWriter pw) {
> +    return super.explainTerms(pw).item("directscan",
> groupScan.getDigest());
> +  }
> +
> +  @Override
> +  public double getRows() {
> +    final PlannerSettings settings =
> PrelUtil.getPlannerSettings(getCluster());
> +    return groupScan.getScanStats(settings).getRecordCount();
> +  }
> +
> +  public DirectGroupScan getGroupScan() {
> +    return groupScan;
> +  }
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/drill/blob/5dbaafbe/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DirectScanPrule.java
> ----------------------------------------------------------------------
> diff --git
>
> a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DirectScanPrule.java
>
> b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DirectScanPrule.java
> new file mode 100644
> index 0000000..5c2fd29
> --- /dev/null
> +++
>
> b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DirectScanPrule.java
> @@ -0,0 +1,49 @@
> +/**
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements.  See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership.  The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License.  You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +package org.apache.drill.exec.planner.physical;
> +
> +import org.apache.calcite.plan.RelOptRule;
> +import org.apache.calcite.plan.RelOptRuleCall;
> +import org.apache.calcite.plan.RelTraitSet;
> +import org.apache.drill.exec.planner.logical.DrillDirectScanRel;
> +import org.apache.drill.exec.planner.logical.RelOptHelper;
> +
> +public class DirectScanPrule extends Prule {
> +
> +  public static final RelOptRule INSTANCE = new DirectScanPrule();
> +
> +  public DirectScanPrule() {
> +    super(RelOptHelper.any(DrillDirectScanRel.class),
> "Prel.DirectScanPrule");
> +  }
> +
> +  @Override
> +  public void onMatch(RelOptRuleCall call) {
> +    final DrillDirectScanRel scan = call.rel(0);
> +    final RelTraitSet traits =
> scan.getTraitSet().plus(Prel.DRILL_PHYSICAL);
> +
> +    final ScanPrel newScan = new ScanPrel(scan.getCluster(), traits,
> scan.getGroupScan(), scan.getRowType()) {
> +      // direct scan (no execution) => no accidental column shuffling =>
> no reordering
> +      @Override
> +      public boolean needsFinalColumnReordering() {
> +        return false;
> +      }
> +    };
> +
> +    call.transformTo(newScan);
> +  }
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/drill/blob/5dbaafbe/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
> ----------------------------------------------------------------------
> diff --git
>
> a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
>
> b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
> index 4ca9fe4..341bae2 100644
> ---
>
> a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
> +++
>
> b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
> @@ -206,6 +206,18 @@ public class DefaultSqlHandler extends
> AbstractSqlHandler {
>     * @throws RelConversionException
>     */
>    protected DrillRel convertToDrel(final RelNode relNode) throws
> SqlUnsupportedException, RelConversionException {
> +    if (context.getOptions().getOption(ExecConstants.EARLY_LIMIT0_OPT) &&
> +        context.getPlannerSettings().isTypeInferenceEnabled() &&
> +        FindLimit0Visitor.containsLimit0(relNode)) {
> +      // disable distributed mode
> +      context.getPlannerSettings().forceSingleMode();
> +      // if the schema is known, return the schema directly
> +      final DrillRel shorterPlan;
> +      if ((shorterPlan =
> FindLimit0Visitor.getDirectScanRelIfFullySchemaed(relNode)) != null) {
> +        return shorterPlan;
> +      }
> +    }
> +
>      try {
>        final RelNode convertedRelNode;
>
>
>
> http://git-wip-us.apache.org/repos/asf/drill/blob/5dbaafbe/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/FindLimit0Visitor.java
> ----------------------------------------------------------------------
> diff --git
>
> a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/FindLimit0Visitor.java
>
> b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/FindLimit0Visitor.java
> index d2c5fa6..fa1fe07 100644
> ---
>
> a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/FindLimit0Visitor.java
> +++
>
> b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/FindLimit0Visitor.java
> @@ -17,6 +17,10 @@
>   */
>  package org.apache.drill.exec.planner.sql.handlers;
>
> +import com.google.common.base.Preconditions;
> +import com.google.common.collect.ImmutableSet;
> +import com.google.common.collect.Lists;
> +import org.apache.calcite.plan.RelTraitSet;
>  import org.apache.calcite.rel.RelNode;
>  import org.apache.calcite.rel.RelShuttleImpl;
>  import org.apache.calcite.rel.logical.LogicalAggregate;
> @@ -25,10 +29,27 @@ import org.apache.calcite.rel.logical.LogicalJoin;
>  import org.apache.calcite.rel.logical.LogicalMinus;
>  import org.apache.calcite.rel.logical.LogicalSort;
>  import org.apache.calcite.rel.logical.LogicalUnion;
> +import org.apache.calcite.rel.type.RelDataTypeField;
>  import org.apache.calcite.rex.RexLiteral;
>  import org.apache.calcite.rex.RexNode;
>  import org.apache.calcite.sql.SqlKind;
> +import org.apache.calcite.sql.type.SqlTypeName;
> +import org.apache.drill.common.exceptions.ExecutionSetupException;
> +import org.apache.drill.common.types.TypeProtos;
> +import org.apache.drill.exec.exception.SchemaChangeException;
> +import org.apache.drill.exec.expr.TypeHelper;
> +import org.apache.drill.exec.ops.OperatorContext;
> +import org.apache.drill.exec.physical.base.ScanStats;
> +import org.apache.drill.exec.physical.impl.OutputMutator;
> +import org.apache.drill.exec.planner.logical.DrillDirectScanRel;
>  import org.apache.drill.exec.planner.logical.DrillLimitRel;
> +import org.apache.drill.exec.planner.logical.DrillRel;
> +import org.apache.drill.exec.planner.sql.TypeInferenceUtils;
> +import org.apache.drill.exec.record.MaterializedField;
> +import org.apache.drill.exec.store.AbstractRecordReader;
> +import org.apache.drill.exec.store.direct.DirectGroupScan;
> +
> +import java.util.List;
>
>  /**
>   * Visitor that will identify whether the root portion of the RelNode tree
> contains a limit 0 pattern. In this case, we
> @@ -36,16 +57,68 @@ import
> org.apache.drill.exec.planner.logical.DrillLimitRel;
>   * executing a schema-only query.
>   */
>  public class FindLimit0Visitor extends RelShuttleImpl {
> -  private static final org.slf4j.Logger logger =
> org.slf4j.LoggerFactory.getLogger(FindLimit0Visitor.class);
> +//  private static final org.slf4j.Logger logger =
> org.slf4j.LoggerFactory.getLogger(FindLimit0Visitor.class);
> +
> +  // Some types are excluded in this set:
> +  // + DECIMAL type is not fully supported in general.
> +  // + VARBINARY is not fully tested.
> +  // + MAP, ARRAY are currently not exposed to the planner.
> +  // + TINYINT, SMALLINT are defined in the Drill type system but have
> been turned off for now.
> +  // + SYMBOL, MULTISET, DISTINCT, STRUCTURED, ROW, OTHER, CURSOR,
> COLUMN_LIST are Calcite types
> +  //   currently not supported by Drill, nor defined in the Drill type
> list.
> +  // + ANY is the late binding type.
> +  private static final ImmutableSet<SqlTypeName> TYPES =
> +      ImmutableSet.<SqlTypeName>builder()
> +          .add(SqlTypeName.INTEGER, SqlTypeName.BIGINT, SqlTypeName.FLOAT,
> SqlTypeName.DOUBLE,
> +              SqlTypeName.VARCHAR, SqlTypeName.BOOLEAN, SqlTypeName.DATE,
> SqlTypeName.TIME,
> +              SqlTypeName.TIMESTAMP, SqlTypeName.INTERVAL_YEAR_MONTH,
> SqlTypeName.INTERVAL_DAY_TIME,
> +              SqlTypeName.CHAR)
> +          .build();
> +
> +  /**
> +   * If all field types of the given node are {@link #TYPES recognized
> types} and honored by execution, then this
> +   * method returns the tree: DrillDirectScanRel(field types). Otherwise,
> the method returns null.
> +   *
> +   * @param rel calcite logical rel tree
> +   * @return drill logical rel tree
> +   */
> +  public static DrillRel getDirectScanRelIfFullySchemaed(RelNode rel) {
> +    final List<RelDataTypeField> fieldList =
> rel.getRowType().getFieldList();
> +    final List<SqlTypeName> columnTypes = Lists.newArrayList();
> +    final List<TypeProtos.DataMode> dataModes = Lists.newArrayList();
> +
> +    for (final RelDataTypeField field : fieldList) {
> +      final SqlTypeName sqlTypeName = field.getType().getSqlTypeName();
> +      if (!TYPES.contains(sqlTypeName)) {
> +        return null;
> +      } else {
> +        columnTypes.add(sqlTypeName);
> +        dataModes.add(field.getType().isNullable() ?
> +            TypeProtos.DataMode.OPTIONAL : TypeProtos.DataMode.REQUIRED);
> +      }
> +    }
>
> -  private boolean contains = false;
> +    final RelTraitSet traits =
> rel.getTraitSet().plus(DrillRel.DRILL_LOGICAL);
> +    final RelDataTypeReader reader = new
> RelDataTypeReader(rel.getRowType().getFieldNames(), columnTypes,
> +        dataModes);
> +    return new DrillDirectScanRel(rel.getCluster(), traits,
> +        new DirectGroupScan(reader, ScanStats.ZERO_RECORD_TABLE),
> rel.getRowType());
> +  }
>
> +  /**
> +   * Check if the root portion of the tree contains LIMIT(0).
> +   *
> +   * @param rel rel node tree
> +   * @return true if the root portion of the tree contains LIMIT(0)
> +   */
>    public static boolean containsLimit0(RelNode rel) {
>      FindLimit0Visitor visitor = new FindLimit0Visitor();
>      rel.accept(visitor);
>      return visitor.isContains();
>    }
>
> +  private boolean contains = false;
> +
>    private FindLimit0Visitor() {
>    }
>
> @@ -53,7 +126,7 @@ public class FindLimit0Visitor extends RelShuttleImpl {
>      return contains;
>    }
>
> -  private boolean isLimit0(RexNode fetch) {
> +  private static boolean isLimit0(RexNode fetch) {
>      if (fetch != null && fetch.isA(SqlKind.LITERAL)) {
>        RexLiteral l = (RexLiteral) fetch;
>        switch (l.getTypeName()) {
> @@ -116,4 +189,49 @@ public class FindLimit0Visitor extends RelShuttleImpl
> {
>    public RelNode visit(LogicalUnion union) {
>      return union;
>    }
> +
> +  /**
> +   * Reader for column names and types.
> +   */
> +  public static class RelDataTypeReader extends AbstractRecordReader {
> +
> +    public final List<String> columnNames;
> +    public final List<SqlTypeName> columnTypes;
> +    public final List<TypeProtos.DataMode> dataModes;
> +
> +    public RelDataTypeReader(List<String> columnNames, List<SqlTypeName>
> columnTypes,
> +                             List<TypeProtos.DataMode> dataModes) {
> +      Preconditions.checkArgument(columnNames.size() == columnTypes.size()
> &&
> +          columnTypes.size() == dataModes.size());
> +      this.columnNames = columnNames;
> +      this.columnTypes = columnTypes;
> +      this.dataModes = dataModes;
> +    }
> +
> +    @Override
> +    public void setup(OperatorContext context, OutputMutator output)
> throws ExecutionSetupException {
> +      for (int i = 0; i < columnNames.size(); i++) {
> +        final TypeProtos.MajorType type =
> TypeProtos.MajorType.newBuilder()
> +            .setMode(dataModes.get(i))
> +
>
> .setMinorType(TypeInferenceUtils.getDrillTypeFromCalciteType(columnTypes.get(i)))
> +            .build();
> +        final MaterializedField field =
> MaterializedField.create(columnNames.get(i), type);
> +        final Class vvClass =
> TypeHelper.getValueVectorClass(type.getMinorType(), type.getMode());
> +        try {
> +          output.addField(field, vvClass);
> +        } catch (SchemaChangeException e) {
> +          throw new ExecutionSetupException(e);
> +        }
> +      }
> +    }
> +
> +    @Override
> +    public int next() {
> +      return 0;
> +    }
> +
> +    @Override
> +    public void close() throws Exception {
> +    }
> +  }
>  }
>
>
> http://git-wip-us.apache.org/repos/asf/drill/blob/5dbaafbe/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
> ----------------------------------------------------------------------
> diff --git
>
> a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
>
> b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
> index cbc5c09..a596d3a 100644
> ---
>
> a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
> +++
>
> b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
> @@ -116,6 +116,7 @@ public class SystemOptionManager extends
> BaseOptionManager implements AutoClosea
>        ExecConstants.SMALL_QUEUE_SIZE,
>        ExecConstants.MIN_HASH_TABLE_SIZE,
>        ExecConstants.MAX_HASH_TABLE_SIZE,
> +      ExecConstants.EARLY_LIMIT0_OPT,
>        ExecConstants.ENABLE_MEMORY_ESTIMATION,
>        ExecConstants.MAX_QUERY_MEMORY_PER_NODE,
>        ExecConstants.NON_BLOCKING_OPERATORS_MEMORY,
>
>
> http://git-wip-us.apache.org/repos/asf/drill/blob/5dbaafbe/exec/java-exec/src/main/java/org/apache/drill/exec/store/direct/DirectGroupScan.java
> ----------------------------------------------------------------------
> diff --git
>
> a/exec/java-exec/src/main/java/org/apache/drill/exec/store/direct/DirectGroupScan.java
>
> b/exec/java-exec/src/main/java/org/apache/drill/exec/store/direct/DirectGroupScan.java
> index e08fe71..a4b2fad 100644
> ---
>
> a/exec/java-exec/src/main/java/org/apache/drill/exec/store/direct/DirectGroupScan.java
> +++
>
> b/exec/java-exec/src/main/java/org/apache/drill/exec/store/direct/DirectGroupScan.java
> @@ -17,12 +17,9 @@
>   */
>  package org.apache.drill.exec.store.direct;
>
> -import java.util.Collections;
> -import java.util.List;
> -
> +import com.fasterxml.jackson.annotation.JsonTypeName;
>  import org.apache.drill.common.exceptions.ExecutionSetupException;
>  import org.apache.drill.common.expression.SchemaPath;
> -import org.apache.drill.exec.physical.EndpointAffinity;
>  import org.apache.drill.exec.physical.PhysicalOperatorSetupException;
>  import org.apache.drill.exec.physical.base.AbstractGroupScan;
>  import org.apache.drill.exec.physical.base.GroupScan;
> @@ -32,14 +29,23 @@ import org.apache.drill.exec.physical.base.SubScan;
>  import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
>  import org.apache.drill.exec.store.RecordReader;
>
> -public class DirectGroupScan extends AbstractGroupScan{
> -  static final org.slf4j.Logger logger =
> org.slf4j.LoggerFactory.getLogger(DirectGroupScan.class);
> +import java.util.List;
> +
> +@JsonTypeName("direct-scan")
> +public class DirectGroupScan extends AbstractGroupScan {
> +//  private static final org.slf4j.Logger logger =
> org.slf4j.LoggerFactory.getLogger(DirectGroupScan.class);
>
>    private final RecordReader reader;
> +  private final ScanStats stats;
>
>    public DirectGroupScan(RecordReader reader) {
> -    super((String)null);
> +    this(reader, ScanStats.TRIVIAL_TABLE);
> +  }
> +
> +  public DirectGroupScan(RecordReader reader, ScanStats stats) {
> +    super((String) null);
>      this.reader = reader;
> +    this.stats = stats;
>    }
>
>    @Override
> @@ -58,14 +64,15 @@ public class DirectGroupScan extends AbstractGroupScan{
>      return 1;
>    }
>
> -  public ScanStats getScanStats(){
> -    return ScanStats.TRIVIAL_TABLE;
> +  @Override
> +  public ScanStats getScanStats() {
> +    return stats;
>    }
>
>    @Override
>    public PhysicalOperator getNewWithChildren(List<PhysicalOperator>
> children) throws ExecutionSetupException {
>      assert children == null || children.isEmpty();
> -    return new DirectGroupScan(reader);
> +    return new DirectGroupScan(reader, stats);
>    }
>
>    @Override
>
>
> http://git-wip-us.apache.org/repos/asf/drill/blob/5dbaafbe/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java
> ----------------------------------------------------------------------
> diff --git
> a/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java
> b/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java
> index 67017ce..f853414 100644
> --- a/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java
> +++ b/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java
> @@ -31,8 +31,10 @@ import java.util.Map;
>  import java.util.Set;
>  import java.util.TreeMap;
>
> +import org.apache.commons.lang3.tuple.Pair;
>  import org.apache.drill.common.expression.SchemaPath;
>  import org.apache.drill.common.types.TypeProtos;
> +import org.apache.drill.common.types.Types;
>  import org.apache.drill.exec.HyperVectorValueIterator;
>  import org.apache.drill.exec.exception.SchemaChangeException;
>  import org.apache.drill.exec.memory.BufferAllocator;
> @@ -294,9 +296,7 @@ public class DrillTestWrapper {
>
>    protected void compareSchemaOnly() throws Exception {
>      RecordBatchLoader loader = new RecordBatchLoader(getAllocator());
> -    List<QueryDataBatch> actual = Collections.EMPTY_LIST;
> -
> -
> +    List<QueryDataBatch> actual;
>      QueryDataBatch batch = null;
>      try {
>        BaseTestQuery.test(testOptionSettingQueries);
> @@ -305,21 +305,24 @@ public class DrillTestWrapper {
>        loader.load(batch.getHeader().getDef(), batch.getData());
>
>        final BatchSchema schema = loader.getSchema();
> -      if(schema.getFieldCount() != testBuilder.getExpectedSchema().size())
> {
> -        throw new Exception("The column numbers for actual schema and
> expected schema do not match");
> +      final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema =
> testBuilder.getExpectedSchema();
> +      if(schema.getFieldCount() != expectedSchema.size()) {
> +        throw new Exception("Expected and actual numbers of columns do not
> match.");
>        }
>
>        for(int i = 0; i < schema.getFieldCount(); ++i) {
>          final String actualSchemaPath = schema.getColumn(i).getPath();
>          final TypeProtos.MajorType actualMajorType =
> schema.getColumn(i).getType();
>
> -        final String expectedSchemaPath = schema.getColumn(i).getPath();
> -        final TypeProtos.MajorType expectedlMajorType =
> schema.getColumn(i).getType();
> +        final String expectedSchemaPath =
> expectedSchema.get(i).getLeft().getAsUnescapedPath();
> +        final TypeProtos.MajorType expectedMajorType =
> expectedSchema.get(i).getValue();
>
> -        if(!actualSchemaPath.equalsIgnoreCase(expectedSchemaPath)
> -            || !actualMajorType.equals(expectedlMajorType)) {
> -          throw new Exception("The type of the " + i + "-th column is '" +
> actualSchemaPath + "' mismatched, expected: '"
> -              + expectedlMajorType + "'");
> +        if(!actualSchemaPath.equals(expectedSchemaPath)
> +            || !actualMajorType.equals(expectedMajorType)) {
> +          throw new Exception(String.format("Schema path or type mismatch
> for column #%d:\n" +
> +                  "Expected schema path: %s\nActual   schema path:
> %s\nExpected type: %s\nActual   type: %s",
> +              i, expectedSchemaPath, actualSchemaPath,
> Types.toString(expectedMajorType),
> +              Types.toString(actualMajorType)));
>          }
>        }
>
>
>
> http://git-wip-us.apache.org/repos/asf/drill/blob/5dbaafbe/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java
> ----------------------------------------------------------------------
> diff --git
> a/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java
> b/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java
> index 3922a38..bb5ff88 100644
> --- a/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java
> +++ b/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java
> @@ -291,6 +291,7 @@ public class PlanTestBase extends BaseTestQuery {
>      final List<QueryDataBatch> results = testSqlWithResults(sql);
>      final RecordBatchLoader loader = new
> RecordBatchLoader(getDrillbitContext().getAllocator());
>      final StringBuilder builder = new StringBuilder();
> +    final boolean silent = config != null &&
> config.getBoolean(QueryTestUtil.TEST_QUERY_PRINTING_SILENT);
>
>      for (final QueryDataBatch b : results) {
>        if (!b.hasData()) {
> @@ -308,12 +309,16 @@ public class PlanTestBase extends BaseTestQuery {
>          throw new Exception("Looks like you did not provide an explain
> plan query, please add EXPLAIN PLAN FOR to the beginning of your query.");
>        }
>
> -      System.out.println(vw.getValueVector().getField().getPath());
> +      if (!silent) {
> +        System.out.println(vw.getValueVector().getField().getPath());
> +      }
>        final ValueVector vv = vw.getValueVector();
>        for (int i = 0; i < vv.getAccessor().getValueCount(); i++) {
>          final Object o = vv.getAccessor().getObject(i);
>          builder.append(o);
> -        System.out.println(vv.getAccessor().getObject(i));
> +        if (!silent) {
> +          System.out.println(o);
> +        }
>        }
>        loader.clear();
>        b.release();
>
>
> http://git-wip-us.apache.org/repos/asf/drill/blob/5dbaafbe/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestEarlyLimit0Optimization.java
> ----------------------------------------------------------------------
> diff --git
>
> a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestEarlyLimit0Optimization.java
>
> b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestEarlyLimit0Optimization.java
> new file mode 100644
> index 0000000..70b0cb3
> --- /dev/null
> +++
>
> b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestEarlyLimit0Optimization.java
> @@ -0,0 +1,663 @@
> +/**
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements.  See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership.  The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License.  You may obtain a copy of the License at
> + * <p/>
> + * http://www.apache.org/licenses/LICENSE-2.0
> + * <p/>
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +package org.apache.drill.exec.physical.impl.limit;
> +
> +import com.google.common.collect.Lists;
> +import org.apache.commons.lang3.tuple.Pair;
> +import org.apache.drill.BaseTestQuery;
> +import org.apache.drill.PlanTestBase;
> +import org.apache.drill.common.expression.SchemaPath;
> +import org.apache.drill.common.types.TypeProtos;
> +import org.apache.drill.common.types.Types;
> +import org.apache.drill.exec.ExecConstants;
> +import org.joda.time.DateTime;
> +import org.junit.After;
> +import org.junit.AfterClass;
> +import org.junit.Before;
> +import org.junit.BeforeClass;
> +import org.junit.Test;
> +
> +import java.sql.Date;
> +import java.util.List;
> +
> +public class TestEarlyLimit0Optimization extends BaseTestQuery {
> +
> +  private static final String viewName = "limitZeroEmployeeView";
> +
> +  private static String wrapLimit0(final String query) {
> +    return "SELECT * FROM (" + query + ") LZT LIMIT 0";
> +  }
> +
> +  @BeforeClass
> +  public static void createView() throws Exception {
> +    test("USE dfs_test.tmp");
> +    test(String.format("CREATE OR REPLACE VIEW %s AS SELECT " +
> +        "CAST(employee_id AS INT) AS employee_id, " +
> +        "CAST(full_name AS VARCHAR(25)) AS full_name, " +
> +        "CAST(position_id AS INTEGER) AS position_id, " +
> +        "CAST(department_id AS BIGINT) AS department_id," +
> +        "CAST(birth_date AS DATE) AS birth_date, " +
> +        "CAST(hire_date AS TIMESTAMP) AS hire_date, " +
> +        "CAST(salary AS DOUBLE) AS salary, " +
> +        "CAST(salary AS FLOAT) AS fsalary, " +
> +        "CAST((CASE WHEN marital_status = 'S' THEN true ELSE false END) AS
> BOOLEAN) AS single, " +
> +        "CAST(education_level AS VARCHAR(60)) AS education_level," +
> +        "CAST(gender AS CHAR) AS gender " +
> +        "FROM cp.`employee.json` " +
> +        "ORDER BY employee_id " +
> +        "LIMIT 1;", viewName));
> +    // { "employee_id":1,"full_name":"Sheri
> Nowmer","first_name":"Sheri","last_name":"Nowmer","position_id":1,
> +    //
>
> "position_title":"President","store_id":0,"department_id":1,"birth_date":"1961-08-26",
> +    // "hire_date":"1994-12-01
> 00:00:00.0","end_date":null,"salary":80000.0000,"supervisor_id":0,
> +    // "education_level":"Graduate
> Degree","marital_status":"S","gender":"F","management_role":"Senior
> Management" }
> +  }
> +
> +  @AfterClass
> +  public static void tearDownView() throws Exception {
> +    test("DROP VIEW " + viewName + ";");
> +  }
> +
> +  @Before
> +  public void setOption() throws Exception {
> +    test("SET `%s` = true;", ExecConstants.EARLY_LIMIT0_OPT_KEY);
> +  }
> +
> +  @After
> +  public void resetOption() throws Exception {
> +    test("RESET `%s`;", ExecConstants.EARLY_LIMIT0_OPT_KEY);
> +  }
> +
> +  // -------------------- SIMPLE QUERIES --------------------
> +
> +  @Test
> +  public void infoSchema() throws Exception {
> +    testBuilder()
> +        .sqlQuery(String.format("DESCRIBE %s", viewName))
> +        .unOrdered()
> +        .baselineColumns("COLUMN_NAME", "DATA_TYPE", "IS_NULLABLE")
> +        .baselineValues("employee_id", "INTEGER", "YES")
> +        .baselineValues("full_name", "CHARACTER VARYING", "YES")
> +        .baselineValues("position_id", "INTEGER", "YES")
> +        .baselineValues("department_id", "BIGINT", "YES")
> +        .baselineValues("birth_date", "DATE", "YES")
> +        .baselineValues("hire_date", "TIMESTAMP", "YES")
> +        .baselineValues("salary", "DOUBLE", "YES")
> +        .baselineValues("fsalary", "FLOAT", "YES")
> +        .baselineValues("single", "BOOLEAN", "NO")
> +        .baselineValues("education_level", "CHARACTER VARYING", "YES")
> +        .baselineValues("gender", "CHARACTER", "YES")
> +        .go();
> +  }
> +
> +  @Test
> +  public void simpleSelect() throws Exception {
> +    testBuilder()
> +        .sqlQuery(String.format("SELECT * FROM %s", viewName))
> +        .ordered()
> +        .baselineColumns("employee_id", "full_name", "position_id",
> "department_id", "birth_date", "hire_date",
> +            "salary", "fsalary", "single", "education_level", "gender")
> +        .baselineValues(1, "Sheri Nowmer", 1, 1L, new
> DateTime(Date.valueOf("1961-08-26").getTime()),
> +            new DateTime(Date.valueOf("1994-12-01").getTime()), 80000.0D,
> 80000.0F, true, "Graduate Degree", "F")
> +        .go();
> +  }
> +
> +  @Test
> +  public void simpleSelectLimit0() throws Exception {
> +    @SuppressWarnings("unchecked")
> +    final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema =
> Lists.newArrayList(
> +        Pair.of(SchemaPath.getSimplePath("employee_id"), Types.optional(
> TypeProtos.MinorType.INT)),
> +        Pair.of(SchemaPath.getSimplePath("full_name"),
> Types.optional(TypeProtos.MinorType.VARCHAR)),
> +        Pair.of(SchemaPath.getSimplePath("position_id"), Types.optional(
> TypeProtos.MinorType.INT)),
> +        Pair.of(SchemaPath.getSimplePath("department_id"),
> Types.optional(TypeProtos.MinorType.BIGINT)),
> +        Pair.of(SchemaPath.getSimplePath("birth_date"),
> Types.optional(TypeProtos.MinorType.DATE)),
> +        Pair.of(SchemaPath.getSimplePath("hire_date"),
> Types.optional(TypeProtos.MinorType.TIMESTAMP)),
> +        Pair.of(SchemaPath.getSimplePath("salary"),
> Types.optional(TypeProtos.MinorType.FLOAT8)),
> +        Pair.of(SchemaPath.getSimplePath("fsalary"),
> Types.optional(TypeProtos.MinorType.FLOAT4)),
> +        Pair.of(SchemaPath.getSimplePath("single"),
> Types.required(TypeProtos.MinorType.BIT)),
> +        Pair.of(SchemaPath.getSimplePath("education_level"),
> Types.optional(TypeProtos.MinorType.VARCHAR)),
> +        Pair.of(SchemaPath.getSimplePath("gender"),
> Types.optional(TypeProtos.MinorType.VARCHAR)));
> +
> +    testBuilder()
> +        .sqlQuery(wrapLimit0(String.format("SELECT * FROM %s", viewName)))
> +        .schemaBaseLine(expectedSchema)
> +        .go();
> +
> +    checkThatQueryPlanIsOptimized("SELECT * FROM " + viewName);
> +  }
> +
> +  private static void checkThatQueryPlanIsOptimized(final String query)
> throws Exception {
> +    PlanTestBase.testPlanMatchingPatterns(
> +        wrapLimit0(query),
> +        new String[]{
> +            ".*Project.*\n" +
> +                ".*Scan.*RelDataTypeReader.*"
> +        },
> +        new String[]{});
> +  }
> +
> +  // -------------------- AGGREGATE FUNC. QUERIES --------------------
> +
> +  private static String getAggQuery(final String functionName) {
> +    return "SELECT " +
> +        functionName + "(employee_id) AS e, " +
> +        functionName + "(position_id) AS p, " +
> +        functionName + "(department_id) AS d, " +
> +        functionName + "(salary) AS s, " +
> +        functionName + "(fsalary) AS f " +
> +        "FROM " + viewName;
> +  }
> +
> +  @Test
> +  public void sums() throws Exception {
> +    final String query = getAggQuery("SUM");
> +
> +    @SuppressWarnings("unchecked")
> +    final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema =
> Lists.newArrayList(
> +        Pair.of(SchemaPath.getSimplePath("e"),
> Types.optional(TypeProtos.MinorType.BIGINT)),
> +        Pair.of(SchemaPath.getSimplePath("p"),
> Types.optional(TypeProtos.MinorType.BIGINT)),
> +        Pair.of(SchemaPath.getSimplePath("d"),
> Types.optional(TypeProtos.MinorType.BIGINT)),
> +        Pair.of(SchemaPath.getSimplePath("s"),
> Types.optional(TypeProtos.MinorType.FLOAT8)),
> +        Pair.of(SchemaPath.getSimplePath("f"),
> Types.optional(TypeProtos.MinorType.FLOAT8)));
> +
> +    testBuilder()
> +        .sqlQuery(query)
> +        .ordered()
> +        .baselineColumns("e", "p", "d", "s", "f")
> +        .baselineValues(1L, 1L, 1L, 80000D, 80000D)
> +        .go();
> +
> +    testBuilder()
> +        .sqlQuery(wrapLimit0(query))
> +        .schemaBaseLine(expectedSchema)
> +        .go();
> +
> +    checkThatQueryPlanIsOptimized(query);
> +  }
> +
> +  @Test
> +  public void counts() throws Exception {
> +    final String query = getAggQuery("COUNT");
> +
> +    @SuppressWarnings("unchecked")
> +    final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema =
> Lists.newArrayList(
> +        Pair.of(SchemaPath.getSimplePath("e"),
> Types.required(TypeProtos.MinorType.BIGINT)),
> +        Pair.of(SchemaPath.getSimplePath("p"),
> Types.required(TypeProtos.MinorType.BIGINT)),
> +        Pair.of(SchemaPath.getSimplePath("d"),
> Types.required(TypeProtos.MinorType.BIGINT)),
> +        Pair.of(SchemaPath.getSimplePath("s"),
> Types.required(TypeProtos.MinorType.BIGINT)),
> +        Pair.of(SchemaPath.getSimplePath("f"),
> Types.required(TypeProtos.MinorType.BIGINT)));
> +
> +    testBuilder()
> +        .sqlQuery(query)
> +        .baselineColumns("e", "p", "d", "s", "f")
> +        .ordered()
> +        .baselineValues(1L, 1L, 1L, 1L, 1L)
> +        .go();
> +
> +    testBuilder()
> +        .sqlQuery(wrapLimit0(query))
> +        .schemaBaseLine(expectedSchema)
> +        .go();
> +
> +    checkThatQueryPlanIsOptimized(query);
> +  }
> +
> +  private void minAndMaxTest(final String functionName) throws Exception {
> +    final String query = getAggQuery(functionName);
> +
> +    @SuppressWarnings("unchecked")
> +    final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema =
> Lists.newArrayList(
> +        Pair.of(SchemaPath.getSimplePath("e"), Types.optional(
> TypeProtos.MinorType.INT)),
> +        Pair.of(SchemaPath.getSimplePath("p"), Types.optional(
> TypeProtos.MinorType.INT)),
> +        Pair.of(SchemaPath.getSimplePath("d"),
> Types.optional(TypeProtos.MinorType.BIGINT)),
> +        Pair.of(SchemaPath.getSimplePath("s"),
> Types.optional(TypeProtos.MinorType.FLOAT8)),
> +        Pair.of(SchemaPath.getSimplePath("f"),
> Types.optional(TypeProtos.MinorType.FLOAT4)));
> +
> +    testBuilder()
> +        .sqlQuery(query)
> +        .baselineColumns("e", "p", "d", "s", "f")
> +        .ordered()
> +        .baselineValues(1, 1, 1L, 80_000D, 80_000F)
> +        .go();
> +
> +    testBuilder()
> +        .sqlQuery(wrapLimit0(query))
> +        .schemaBaseLine(expectedSchema)
> +        .go();
> +
> +    checkThatQueryPlanIsOptimized(query);
> +  }
> +
> +  @Test
> +  public void mins() throws Exception {
> +    minAndMaxTest("MIN");
> +  }
> +
> +  @Test
> +  public void maxs() throws Exception {
> +    minAndMaxTest("MAX");
> +  }
> +
> +  @Test
> +  public void avgs() throws Exception {
> +    final String query = getAggQuery("AVG");
> +
> +    @SuppressWarnings("unchecked")
> +    final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema =
> Lists.newArrayList(
> +        Pair.of(SchemaPath.getSimplePath("e"),
> Types.optional(TypeProtos.MinorType.FLOAT8)),
> +        Pair.of(SchemaPath.getSimplePath("p"),
> Types.optional(TypeProtos.MinorType.FLOAT8)),
> +        Pair.of(SchemaPath.getSimplePath("d"),
> Types.optional(TypeProtos.MinorType.FLOAT8)),
> +        Pair.of(SchemaPath.getSimplePath("s"),
> Types.optional(TypeProtos.MinorType.FLOAT8)),
> +        Pair.of(SchemaPath.getSimplePath("f"),
> Types.optional(TypeProtos.MinorType.FLOAT8)));
> +
> +    testBuilder()
> +        .sqlQuery(query)
> +        .ordered()
> +        .baselineColumns("e", "p", "d", "s", "f")
> +        .baselineValues(1D, 1D, 1D, 80_000D, 80_000D)
> +        .go();
> +
> +    testBuilder()
> +        .sqlQuery(wrapLimit0(query))
> +        .schemaBaseLine(expectedSchema)
> +        .go();
> +
> +    checkThatQueryPlanIsOptimized(query);
> +  }
> +
> +  @Test
> +  public void measures() throws Exception {
> +    final String query = "SELECT " +
> +        "STDDEV_SAMP(employee_id) AS s, " +
> +        "STDDEV_POP(position_id) AS p, " +
> +        "AVG(position_id) AS a, " +
> +        "COUNT(position_id) AS c " +
> +        "FROM " + viewName;
> +
> +    @SuppressWarnings("unchecked")
> +    final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema =
> Lists.newArrayList(
> +        Pair.of(SchemaPath.getSimplePath("s"),
> Types.optional(TypeProtos.MinorType.FLOAT8)),
> +        Pair.of(SchemaPath.getSimplePath("p"),
> Types.optional(TypeProtos.MinorType.FLOAT8)),
> +        Pair.of(SchemaPath.getSimplePath("a"),
> Types.optional(TypeProtos.MinorType.FLOAT8)),
> +        Pair.of(SchemaPath.getSimplePath("c"),
> Types.required(TypeProtos.MinorType.BIGINT)));
> +
> +    testBuilder()
> +        .sqlQuery(query)
> +        .ordered()
> +        .baselineColumns("s", "p", "a", "c")
> +        .baselineValues(null, 0.0D, 1.0D, 1L)
> +        .go();
> +
> +    testBuilder()
> +        .sqlQuery(wrapLimit0(query))
> +        .schemaBaseLine(expectedSchema)
> +        .go();
> +
> +    checkThatQueryPlanIsOptimized(query);
> +  }
> +
> +  @Test
> +  public void nullableCount() throws Exception {
> +    final String query = "SELECT " +
> +        "COUNT(CASE WHEN position_id = 1 THEN NULL ELSE position_id END)
> AS c FROM " + viewName;
> +
> +    @SuppressWarnings("unchecked")
> +    final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema =
> Lists.newArrayList(
> +        Pair.of(SchemaPath.getSimplePath("c"),
> Types.required(TypeProtos.MinorType.BIGINT)));
> +
> +    testBuilder()
> +        .sqlQuery(query)
> +        .ordered()
> +        .baselineColumns("c")
> +        .baselineValues(0L)
> +        .go();
> +
> +    testBuilder()
> +        .sqlQuery(wrapLimit0(query))
> +        .schemaBaseLine(expectedSchema)
> +        .go();
> +
> +    checkThatQueryPlanIsOptimized(query);
> +  }
> +
> +  @Test
> +  public void nullableSumAndCount() throws Exception {
> +    final String query = "SELECT " +
> +        "COUNT(position_id) AS c, " +
> +        "SUM(CAST((CASE WHEN position_id = 1 THEN NULL ELSE position_id
> END) AS INT)) AS p " +
> +        "FROM " + viewName;
> +
> +    @SuppressWarnings("unchecked")
> +    final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema =
> Lists.newArrayList(
> +        Pair.of(SchemaPath.getSimplePath("c"),
> Types.required(TypeProtos.MinorType.BIGINT)),
> +        Pair.of(SchemaPath.getSimplePath("p"),
> Types.optional(TypeProtos.MinorType.BIGINT)));
> +
> +    testBuilder()
> +        .sqlQuery(query)
> +        .ordered()
> +        .baselineColumns("c", "p")
> +        .baselineValues(1L, null)
> +        .go();
> +
> +    testBuilder()
> +        .sqlQuery(wrapLimit0(query))
> +        .schemaBaseLine(expectedSchema)
> +        .go();
> +
> +    checkThatQueryPlanIsOptimized(query);
> +  }
> +
> +  @Test
> +  public void castSum() throws Exception {
> +    final String query = "SELECT CAST(SUM(position_id) AS INT) AS s FROM
> cp.`employee.json`";
> +
> +    @SuppressWarnings("unchecked")
> +    final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema =
> Lists.newArrayList(
> +        Pair.of(SchemaPath.getSimplePath("s"), Types.optional(
> TypeProtos.MinorType.INT)));
> +
> +    testBuilder()
> +        .sqlQuery(query)
> +        .ordered()
> +        .baselineColumns("s")
> +        .baselineValues(18422)
> +        .go();
> +
> +    testBuilder()
> +        .sqlQuery(wrapLimit0(query))
> +        .schemaBaseLine(expectedSchema)
> +        .go();
> +
> +    checkThatQueryPlanIsOptimized(query);
> +  }
> +
> +  @Test
> +  public void sumCast() throws Exception {
> +    final String query = "SELECT SUM(CAST(position_id AS INT)) AS s FROM
> cp.`employee.json`";
> +
> +    @SuppressWarnings("unchecked")
> +    final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema =
> Lists.newArrayList(
> +        Pair.of(SchemaPath.getSimplePath("s"),
> Types.optional(TypeProtos.MinorType.BIGINT)));
> +
> +    testBuilder()
> +        .sqlQuery(query)
> +        .ordered()
> +        .baselineColumns("s")
> +        .baselineValues(18422L)
> +        .go();
> +
> +    testBuilder()
> +        .sqlQuery(wrapLimit0(query))
> +        .schemaBaseLine(expectedSchema)
> +        .go();
> +
> +    checkThatQueryPlanIsOptimized(query);
> +  }
> +
> +  @Test
> +  public void sumsAndCounts1() throws Exception {
> +    final String query = "SELECT " +
> +        "COUNT(*) as cs, " +
> +        "COUNT(1) as c1, " +
> +        "COUNT(employee_id) as cc, " +
> +        "SUM(1) as s1," +
> +        "department_id " +
> +        " FROM " + viewName + " GROUP BY department_id";
> +
> +    @SuppressWarnings("unchecked")
> +    final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema =
> Lists.newArrayList(
> +        Pair.of(SchemaPath.getSimplePath("cs"),
> Types.required(TypeProtos.MinorType.BIGINT)),
> +        Pair.of(SchemaPath.getSimplePath("c1"),
> Types.required(TypeProtos.MinorType.BIGINT)),
> +        Pair.of(SchemaPath.getSimplePath("cc"),
> Types.required(TypeProtos.MinorType.BIGINT)),
> +        Pair.of(SchemaPath.getSimplePath("s1"),
> Types.required(TypeProtos.MinorType.BIGINT)),
> +        Pair.of(SchemaPath.getSimplePath("department_id"),
> Types.optional(TypeProtos.MinorType.BIGINT)));
> +
> +    testBuilder()
> +        .sqlQuery(query)
> +        .ordered()
> +        .baselineColumns("cs", "c1", "cc", "s1", "department_id")
> +        .baselineValues(1L, 1L, 1L, 1L, 1L)
> +        .go();
> +
> +    testBuilder()
> +        .sqlQuery(wrapLimit0(query))
> +        .schemaBaseLine(expectedSchema)
> +        .go();
> +
> +    checkThatQueryPlanIsOptimized(query);
> +  }
> +
> +  @Test
> +  public void sumsAndCounts2() throws Exception {
> +    final String query = "SELECT " +
> +        "SUM(1) as s1, " +
> +        "COUNT(1) as c1, " +
> +        "COUNT(*) as cs, " +
> +        "COUNT(CAST(n_regionkey AS INT)) as cc " +
> +        "FROM cp.`tpch/nation.parquet` " +
> +        "GROUP BY CAST(n_regionkey AS INT)";
> +
> +    @SuppressWarnings("unchecked")
> +    final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema =
> Lists.newArrayList(
> +        Pair.of(SchemaPath.getSimplePath("s1"),
> Types.required(TypeProtos.MinorType.BIGINT)),
> +        Pair.of(SchemaPath.getSimplePath("c1"),
> Types.required(TypeProtos.MinorType.BIGINT)),
> +        Pair.of(SchemaPath.getSimplePath("cs"),
> Types.required(TypeProtos.MinorType.BIGINT)),
> +        Pair.of(SchemaPath.getSimplePath("cc"),
> Types.required(TypeProtos.MinorType.BIGINT)));
> +
> +    testBuilder()
> +        .sqlQuery(query)
> +        .ordered()
> +        .baselineColumns("s1", "c1", "cs", "cc")
> +        .baselineValues(5L, 5L, 5L, 5L)
> +        .baselineValues(5L, 5L, 5L, 5L)
> +        .baselineValues(5L, 5L, 5L, 5L)
> +        .baselineValues(5L, 5L, 5L, 5L)
> +        .baselineValues(5L, 5L, 5L, 5L)
> +        .go();
> +
> +    testBuilder()
> +        .sqlQuery(wrapLimit0(query))
> +        .schemaBaseLine(expectedSchema)
> +        .go();
> +
> +    checkThatQueryPlanIsOptimized(query);
> +
> +  }
> +
> +  @Test
> +  public void rank() throws Exception {
> +    final String query = "SELECT RANK() OVER(PARTITION BY employee_id
> ORDER BY employee_id) AS r FROM " + viewName;
> +
> +    @SuppressWarnings("unchecked")
> +    final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema =
> Lists.newArrayList(
> +        Pair.of(SchemaPath.getSimplePath("r"),
> Types.required(TypeProtos.MinorType.BIGINT)));
> +
> +    testBuilder()
> +        .sqlQuery(query)
> +        .ordered()
> +        .baselineColumns("r")
> +        .baselineValues(1L)
> +        .go();
> +
> +    testBuilder()
> +        .sqlQuery(wrapLimit0(query))
> +        .schemaBaseLine(expectedSchema)
> +        .go();
> +
> +    checkThatQueryPlanIsOptimized(query);
> +  }
> +
> +  // -------------------- SCALAR FUNC. QUERIES --------------------
> +
> +  @Test
> +  public void cast() throws Exception {
> +    final String query = "SELECT CAST(fsalary AS DOUBLE) AS d," +
> +        "CAST(employee_id AS BIGINT) AS e FROM " + viewName;
> +
> +    @SuppressWarnings("unchecked")
> +    final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema =
> Lists.newArrayList(
> +        Pair.of(SchemaPath.getSimplePath("d"),
> Types.optional(TypeProtos.MinorType.FLOAT8)),
> +        Pair.of(SchemaPath.getSimplePath("e"),
> Types.optional(TypeProtos.MinorType.BIGINT)));
> +
> +    testBuilder()
> +        .sqlQuery(query)
> +        .baselineColumns("d", "e")
> +        .ordered()
> +        .baselineValues(80_000D, 1L)
> +        .go();
> +
> +    testBuilder()
> +        .sqlQuery(wrapLimit0(query))
> +        .schemaBaseLine(expectedSchema)
> +        .go();
> +
> +    checkThatQueryPlanIsOptimized(query);
> +  }
> +
> +  public void concatTest(final String query) throws Exception {
> +    @SuppressWarnings("unchecked")
> +    final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema =
> Lists.newArrayList(
> +        Pair.of(SchemaPath.getSimplePath("c"),
> Types.optional(TypeProtos.MinorType.VARCHAR)));
> +
> +    testBuilder()
> +        .sqlQuery(query)
> +        .baselineColumns("c")
> +        .ordered()
> +        .baselineValues("Sheri NowmerGraduate Degree")
> +        .go();
> +
> +    testBuilder()
> +        .sqlQuery(wrapLimit0(query))
> +        .schemaBaseLine(expectedSchema)
> +        .go();
> +
> +    checkThatQueryPlanIsOptimized(query);
> +  }
> +
> +  @Test
> +  public void concat() throws Exception {
> +    concatTest("SELECT CONCAT(full_name, education_level) AS c FROM " +
> viewName);
> +  }
> +
> +  @Test
> +  public void concatOp() throws Exception {
> +    concatTest("SELECT full_name || education_level AS c FROM " +
> viewName);
> +  }
> +
> +  @Test
> +  public void extract() throws Exception {
> +    final String query = "SELECT EXTRACT(YEAR FROM hire_date) AS e FROM "
> + viewName;
> +
> +    @SuppressWarnings("unchecked")
> +    final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema =
> Lists.newArrayList(
> +        Pair.of(SchemaPath.getSimplePath("e"),
> Types.optional(TypeProtos.MinorType.BIGINT)));
> +
> +    testBuilder()
> +        .sqlQuery(query)
> +        .baselineColumns("e")
> +        .ordered()
> +        .baselineValues(1994L)
> +        .go();
> +
> +    testBuilder()
> +        .sqlQuery(wrapLimit0(query))
> +        .schemaBaseLine(expectedSchema)
> +        .go();
> +
> +    checkThatQueryPlanIsOptimized(query);
> +  }
> +
> +  @Test
> +  public void binary() throws Exception {
> +    final String query = "SELECT " +
> +        "single AND true AS b, " +
> +        "full_name || education_level AS c, " +
> +        "position_id / position_id AS d, " +
> +        "position_id = position_id AS e, " +
> +        "position_id > position_id AS g, " +
> +        "position_id >= position_id AS ge, " +
> +        "position_id IN (0, 1) AS i, +" +
> +        "position_id < position_id AS l, " +
> +        "position_id <= position_id AS le, " +
> +        "position_id - position_id AS m, " +
> +        "position_id * position_id AS mu, " +
> +        "position_id <> position_id AS n, " +
> +        "single OR false AS o, " +
> +        "position_id + position_id AS p FROM " + viewName;
> +
> +    @SuppressWarnings("unchecked")
> +    final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema =
> Lists.newArrayList(
> +        Pair.of(SchemaPath.getSimplePath("b"),
> Types.required(TypeProtos.MinorType.BIT)),
> +        Pair.of(SchemaPath.getSimplePath("c"),
> Types.optional(TypeProtos.MinorType.VARCHAR)),
> +        Pair.of(SchemaPath.getSimplePath("d"), Types.optional(
> TypeProtos.MinorType.INT)),
> +        Pair.of(SchemaPath.getSimplePath("e"),
> Types.optional(TypeProtos.MinorType.BIT)),
> +        Pair.of(SchemaPath.getSimplePath("g"),
> Types.optional(TypeProtos.MinorType.BIT)),
> +        Pair.of(SchemaPath.getSimplePath("ge"),
> Types.optional(TypeProtos.MinorType.BIT)),
> +        Pair.of(SchemaPath.getSimplePath("i"),
> Types.optional(TypeProtos.MinorType.BIT)),
> +        Pair.of(SchemaPath.getSimplePath("l"),
> Types.optional(TypeProtos.MinorType.BIT)),
> +        Pair.of(SchemaPath.getSimplePath("le"),
> Types.optional(TypeProtos.MinorType.BIT)),
> +        Pair.of(SchemaPath.getSimplePath("m"), Types.optional(
> TypeProtos.MinorType.INT)),
> +        Pair.of(SchemaPath.getSimplePath("mu"), Types.optional(
> TypeProtos.MinorType.INT)),
> +        Pair.of(SchemaPath.getSimplePath("n"),
> Types.optional(TypeProtos.MinorType.BIT)),
> +        Pair.of(SchemaPath.getSimplePath("o"),
> Types.required(TypeProtos.MinorType.BIT)),
> +        Pair.of(SchemaPath.getSimplePath("p"), Types.optional(
> TypeProtos.MinorType.INT)));
> +
> +    testBuilder()
> +        .sqlQuery(query)
> +        .baselineColumns("b", "c", "d", "e", "g", "ge", "i", "l", "le",
> "m", "mu", "n", "o", "p")
> +        .ordered()
> +        .baselineValues(true, "Sheri NowmerGraduate Degree", 1, true,
> false, true, true, false, true,
> +            0, 1, false, true, 2)
> +        .go();
> +
> +    testBuilder()
> +        .sqlQuery(wrapLimit0(query))
> +        .schemaBaseLine(expectedSchema)
> +        .go();
> +
> +    checkThatQueryPlanIsOptimized(query);
> +  }
> +
> +  public void substringTest(final String query) throws Exception {
> +    @SuppressWarnings("unchecked")
> +    final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema =
> Lists.newArrayList(
> +        Pair.of(SchemaPath.getSimplePath("s"),
> Types.optional(TypeProtos.MinorType.VARCHAR)));
> +
> +    testBuilder()
> +        .sqlQuery(query)
> +        .baselineColumns("s")
> +        .ordered()
> +        .baselineValues("Sheri")
> +        .go();
> +
> +    testBuilder()
> +        .sqlQuery(wrapLimit0(query))
> +        .schemaBaseLine(expectedSchema)
> +        .go();
> +
> +    checkThatQueryPlanIsOptimized(query);
> +  }
> +
> +  @Test
> +  public void substring() throws Exception {
> +    substringTest("SELECT SUBSTRING(full_name, 1, 5) AS s FROM " +
> viewName);
> +  }
> +
> +  @Test
> +  public void substr() throws Exception {
> +    substringTest("SELECT SUBSTR(full_name, 1, 5) AS s FROM " + viewName);
> +  }
> +}
>

Mime
  • Unnamed multipart/alternative (inline, None, 0 bytes)
View raw message