hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject hive git commit: HIVE-19586 : Optimize Count(distinct X) pushdown based on the storage capabilities (Slim Bouguerra via Ashutosh Chauhan)
Date Sun, 03 Jun 2018 16:15:46 GMT
Repository: hive
Updated Branches:
  refs/heads/branch-3 ef71812bf -> 7ed820f26


HIVE-19586 : Optimize Count(distinct X) pushdown based on the storage capabilities (Slim Bouguerra via Ashutosh Chauhan)

Signed-off-by: Ashutosh Chauhan <hashutosh@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7ed820f2
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7ed820f2
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7ed820f2

Branch: refs/heads/branch-3
Commit: 7ed820f2602761e7df1962127e55ca0a7d39f19c
Parents: ef71812
Author: Slim Bouguerra <slim.bouguerra@gmail.com>
Authored: Thu May 17 16:40:00 2018 -0700
Committer: Ashutosh Chauhan <hashutosh@apache.org>
Committed: Sun Jun 3 09:15:40 2018 -0700

----------------------------------------------------------------------
 .../optimizer/calcite/rules/HiveDruidRules.java | 240 ++++++++-
 .../hadoop/hive/ql/parse/CalcitePlanner.java    | 104 ++--
 .../clientpositive/druidmini_expressions.q      |  30 ++
 .../druid/druidmini_expressions.q.out           | 521 +++++++++++++++++++
 4 files changed, 832 insertions(+), 63 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/7ed820f2/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveDruidRules.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveDruidRules.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveDruidRules.java
index 7d2b9dc..a914210 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveDruidRules.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveDruidRules.java
@@ -17,40 +17,57 @@
  */
 package org.apache.hadoop.hive.ql.optimizer.calcite.rules;
 
-import org.apache.calcite.adapter.druid.DruidRules.DruidHavingFilterRule;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import org.apache.calcite.adapter.druid.DruidQuery;
 import org.apache.calcite.adapter.druid.DruidRules.DruidAggregateFilterTransposeRule;
 import org.apache.calcite.adapter.druid.DruidRules.DruidAggregateProjectRule;
 import org.apache.calcite.adapter.druid.DruidRules.DruidAggregateRule;
 import org.apache.calcite.adapter.druid.DruidRules.DruidFilterAggregateTransposeRule;
 import org.apache.calcite.adapter.druid.DruidRules.DruidFilterProjectTransposeRule;
 import org.apache.calcite.adapter.druid.DruidRules.DruidFilterRule;
+import org.apache.calcite.adapter.druid.DruidRules.DruidHavingFilterRule;
 import org.apache.calcite.adapter.druid.DruidRules.DruidPostAggregationProjectRule;
 import org.apache.calcite.adapter.druid.DruidRules.DruidProjectFilterTransposeRule;
 import org.apache.calcite.adapter.druid.DruidRules.DruidProjectRule;
 import org.apache.calcite.adapter.druid.DruidRules.DruidProjectSortTransposeRule;
 import org.apache.calcite.adapter.druid.DruidRules.DruidSortProjectTransposeRule;
 import org.apache.calcite.adapter.druid.DruidRules.DruidSortRule;
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.rel.core.Aggregate;
+import org.apache.calcite.rel.core.AggregateCall;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.fun.SqlSumEmptyIsZeroAggFunction;
+import org.apache.calcite.tools.RelBuilder;
+import org.apache.calcite.tools.RelBuilderFactory;
+import org.apache.calcite.util.ImmutableBitSet;
+import org.apache.calcite.util.Pair;
 import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories;
 
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
 /**
  * Druid rules with Hive builder factory.
  */
 public class HiveDruidRules {
 
-  public static final DruidFilterRule FILTER =
-      new DruidFilterRule(HiveRelFactories.HIVE_BUILDER);
+  public static final DruidFilterRule FILTER = new DruidFilterRule(HiveRelFactories.HIVE_BUILDER);
 
-  public static final DruidProjectRule PROJECT =
-      new DruidProjectRule(HiveRelFactories.HIVE_BUILDER);
+  public static final DruidProjectRule PROJECT = new DruidProjectRule(HiveRelFactories.HIVE_BUILDER);
 
-  public static final DruidAggregateRule AGGREGATE =
-      new DruidAggregateRule(HiveRelFactories.HIVE_BUILDER);
+  public static final DruidAggregateRule AGGREGATE = new DruidAggregateRule(HiveRelFactories.HIVE_BUILDER);
 
   public static final DruidAggregateProjectRule AGGREGATE_PROJECT =
       new DruidAggregateProjectRule(HiveRelFactories.HIVE_BUILDER);
 
-  public static final DruidSortRule SORT =
-      new DruidSortRule(HiveRelFactories.HIVE_BUILDER);
+  public static final DruidSortRule SORT = new DruidSortRule(HiveRelFactories.HIVE_BUILDER);
 
   public static final DruidSortProjectTransposeRule SORT_PROJECT_TRANSPOSE =
       new DruidSortProjectTransposeRule(HiveRelFactories.HIVE_BUILDER);
@@ -75,4 +92,209 @@ public class HiveDruidRules {
 
   public static final DruidHavingFilterRule HAVING_FILTER_RULE =
       new DruidHavingFilterRule(HiveRelFactories.HIVE_BUILDER);
+
+  public static final AggregateExpandDistinctAggregatesDruidRule EXPAND_SINGLE_DISTINCT_AGGREGATES_DRUID_RULE =
+      new AggregateExpandDistinctAggregatesDruidRule(HiveRelFactories.HIVE_BUILDER);
+
+  /**
+   * This is a simplified version of {@link org.apache.calcite.rel.rules.AggregateExpandDistinctAggregatesRule}
+   * The goal of this simplified version is to help pushing single count distinct as multi-phase aggregates.
+   * This is an okay solution before we actually support grouping sets push-down to Druid.
+   * We are limiting it to one Distinct count to avoid expensive cross join and running into issue
+   * https://issues.apache.org/jira/browse/HIVE-19601
+   */
+  public static class AggregateExpandDistinctAggregatesDruidRule extends RelOptRule {
+
+    public AggregateExpandDistinctAggregatesDruidRule(RelBuilderFactory relBuilderFactory) {
+      super(operand(Aggregate.class, operand(DruidQuery.class, none())), relBuilderFactory,
+          null
+      );
+    }
+    @Override public void onMatch(RelOptRuleCall call) {
+      Aggregate aggregate =  call.rel(0);
+      if (!aggregate.containsDistinctCall()) {
+        return;
+      }
+      final long numCountDistinct = aggregate.getAggCallList()
+          .stream()
+          .filter(aggregateCall -> aggregateCall.getAggregation().getKind().equals(SqlKind.COUNT) &&
+              aggregateCall.isDistinct())
+          .count();
+      if (numCountDistinct != 1) {
+        return;
+      }
+
+      // Find all of the agg expressions. We use a LinkedHashSet to ensure determinism.
+      int nonDistinctAggCallCount = 0;  // find all aggregate calls without distinct
+      int filterCount = 0;
+      int unsupportedNonDistinctAggCallCount = 0;
+      final Set<Pair<List<Integer>, Integer>> argLists = new LinkedHashSet<>();
+      for (AggregateCall aggCall : aggregate.getAggCallList()) {
+        if (aggCall.filterArg >= 0) {
+          ++filterCount;
+        }
+        if (!aggCall.isDistinct()) {
+          ++nonDistinctAggCallCount;
+          final SqlKind aggCallKind = aggCall.getAggregation().getKind();
+          // We only support COUNT/SUM/MIN/MAX for the "single" count distinct optimization
+          switch (aggCallKind) {
+          case COUNT:
+          case SUM:
+          case SUM0:
+          case MIN:
+          case MAX:
+            break;
+          default:
+            ++unsupportedNonDistinctAggCallCount;
+          }
+        } else {
+          argLists.add(Pair.of(aggCall.getArgList(), aggCall.filterArg));
+        }
+      }
+      // If only one distinct aggregate and one or more non-distinct aggregates,
+      // we can generate multi-phase aggregates
+      if (numCountDistinct == 1 // one distinct aggregate
+          && filterCount == 0 // no filter
+          && unsupportedNonDistinctAggCallCount == 0 // sum/min/max/count in non-distinct aggregate
+          && nonDistinctAggCallCount > 0) { // one or more non-distinct aggregates
+        final RelBuilder relBuilder = call.builder();
+        convertSingletonDistinct(relBuilder, aggregate, argLists);
+        call.transformTo(relBuilder.build());
+        return;
+      }
+    }
+
+    /**
+     * Converts an aggregate with one distinct aggregate and one or more
+     * non-distinct aggregates to multi-phase aggregates (see reference example
+     * below).
+     *
+     * @param relBuilder Contains the input relational expression
+     * @param aggregate  Original aggregate
+     * @param argLists   Arguments and filters to the distinct aggregate function
+     *
+     */
+    private RelBuilder convertSingletonDistinct(RelBuilder relBuilder,
+        Aggregate aggregate, Set<Pair<List<Integer>, Integer>> argLists) {
+
+      // In this case, we are assuming that there is a single distinct function.
+      // So make sure that argLists is of size one.
+      Preconditions.checkArgument(argLists.size() == 1);
+
+      // For example,
+      //    SELECT deptno, COUNT(*), SUM(bonus), MIN(DISTINCT sal)
+      //    FROM emp
+      //    GROUP BY deptno
+      //
+      // becomes
+      //
+      //    SELECT deptno, SUM(cnt), SUM(bonus), MIN(sal)
+      //    FROM (
+      //          SELECT deptno, COUNT(*) as cnt, SUM(bonus), sal
+      //          FROM EMP
+      //          GROUP BY deptno, sal)            // Aggregate B
+      //    GROUP BY deptno                        // Aggregate A
+      relBuilder.push(aggregate.getInput());
+
+      final List<AggregateCall> originalAggCalls = aggregate.getAggCallList();
+      final ImmutableBitSet originalGroupSet = aggregate.getGroupSet();
+
+      // Add the distinct aggregate column(s) to the group-by columns,
+      // if not already a part of the group-by
+      final SortedSet<Integer> bottomGroupSet = new TreeSet<>();
+      bottomGroupSet.addAll(aggregate.getGroupSet().asList());
+      for (AggregateCall aggCall : originalAggCalls) {
+        if (aggCall.isDistinct()) {
+          bottomGroupSet.addAll(aggCall.getArgList());
+          break;  // since we only have single distinct call
+        }
+      }
+
+      // Generate the intermediate aggregate B, the one on the bottom that converts
+      // a distinct call to group by call.
+      // Bottom aggregate is the same as the original aggregate, except that
+      // the bottom aggregate has converted the DISTINCT aggregate to a group by clause.
+      final List<AggregateCall> bottomAggregateCalls = new ArrayList<>();
+      for (AggregateCall aggCall : originalAggCalls) {
+        // Project the column corresponding to the distinct aggregate. Project
+        // as-is all the non-distinct aggregates
+        if (!aggCall.isDistinct()) {
+          final AggregateCall newCall =
+              AggregateCall.create(aggCall.getAggregation(), false,
+                  aggCall.isApproximate(), aggCall.getArgList(), -1,
+                  ImmutableBitSet.of(bottomGroupSet).cardinality(),
+                  relBuilder.peek(), null, aggCall.name);
+          bottomAggregateCalls.add(newCall);
+        }
+      }
+      // Generate the aggregate B (see the reference example above)
+      relBuilder.push(
+          aggregate.copy(
+              aggregate.getTraitSet(), relBuilder.build(),
+              false, ImmutableBitSet.of(bottomGroupSet), null, bottomAggregateCalls));
+
+      // Add aggregate A (see the reference example above), the top aggregate
+      // to handle the rest of the aggregation that the bottom aggregate hasn't handled
+      final List<AggregateCall> topAggregateCalls = Lists.newArrayList();
+      // Use the remapped arguments for the (non)distinct aggregate calls
+      int nonDistinctAggCallProcessedSoFar = 0;
+      for (AggregateCall aggCall : originalAggCalls) {
+        final AggregateCall newCall;
+        if (aggCall.isDistinct()) {
+          List<Integer> newArgList = new ArrayList<>();
+          for (int arg : aggCall.getArgList()) {
+            newArgList.add(bottomGroupSet.headSet(arg).size());
+          }
+          newCall =
+              AggregateCall.create(aggCall.getAggregation(),
+                  false,
+                  aggCall.isApproximate(),
+                  newArgList,
+                  -1,
+                  originalGroupSet.cardinality(),
+                  relBuilder.peek(),
+                  aggCall.getType(),
+                  aggCall.name);
+        } else {
+          // If aggregate B had a COUNT aggregate call the corresponding aggregate at
+          // aggregate A must be SUM. For other aggregates, it remains the same.
+          final List<Integer> newArgs =
+              Lists.newArrayList(bottomGroupSet.size() + nonDistinctAggCallProcessedSoFar);
+          if (aggCall.getAggregation().getKind() == SqlKind.COUNT) {
+            newCall =
+                AggregateCall.create(new SqlSumEmptyIsZeroAggFunction(), false,
+                    aggCall.isApproximate(), newArgs, -1,
+                    originalGroupSet.cardinality(), relBuilder.peek(),
+                    aggCall.getType(), aggCall.getName());
+          } else {
+            newCall =
+                AggregateCall.create(aggCall.getAggregation(), false,
+                    aggCall.isApproximate(), newArgs, -1,
+                    originalGroupSet.cardinality(),
+                    relBuilder.peek(), aggCall.getType(), aggCall.name);
+          }
+          nonDistinctAggCallProcessedSoFar++;
+        }
+
+        topAggregateCalls.add(newCall);
+      }
+
+      // Populate the group-by keys with the remapped arguments for aggregate A
+      // The top groupset is basically an identity (first X fields of aggregate B's
+      // output), minus the distinct aggCall's input.
+      final Set<Integer> topGroupSet = new HashSet<>();
+      int groupSetToAdd = 0;
+      for (int bottomGroup : bottomGroupSet) {
+        if (originalGroupSet.get(bottomGroup)) {
+          topGroupSet.add(groupSetToAdd);
+        }
+        groupSetToAdd++;
+      }
+      relBuilder.push(
+          aggregate.copy(aggregate.getTraitSet(),
+              relBuilder.build(), aggregate.indicator,
+              ImmutableBitSet.of(topGroupSet), null, topAggregateCalls));
+      return relBuilder;
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/7ed820f2/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index 0bc9d23..d939110 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -17,34 +17,14 @@
  */
 package org.apache.hadoop.hive.ql.parse;
 
-import java.io.IOException;
-import java.lang.reflect.Field;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.UndeclaredThrowableException;
-import java.math.BigDecimal;
-import java.util.AbstractMap.SimpleEntry;
-import java.util.ArrayDeque;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.BitSet;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Deque;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import javax.sql.DataSource;
+import com.google.common.base.Function;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableList.Builder;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Multimap;
 import org.antlr.runtime.ClassicToken;
 import org.antlr.runtime.CommonToken;
 import org.antlr.runtime.tree.Tree;
@@ -177,7 +157,6 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveExcept;
 import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter;
 import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveGroupingID;
 import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveIntersect;
-import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.jdbc.HiveJdbcConverter;
 import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin;
 import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
 import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveRelNode;
@@ -186,6 +165,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit;
 import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableFunctionScan;
 import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan;
 import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveUnion;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.jdbc.HiveJdbcConverter;
 import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.jdbc.JdbcHiveTableScan;
 import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveAggregateJoinTransposeRule;
 import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveAggregateProjectMergeRule;
@@ -234,22 +214,19 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveSubQueryRemoveRule;
 import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveUnionMergeRule;
 import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveUnionPullUpConstantsRule;
 import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveWindowingFixRule;
-
+import org.apache.hadoop.hive.ql.optimizer.calcite.rules.jdbc.JDBCAbstractSplitFilterRule;
 import org.apache.hadoop.hive.ql.optimizer.calcite.rules.jdbc.JDBCAggregationPushDownRule;
+import org.apache.hadoop.hive.ql.optimizer.calcite.rules.jdbc.JDBCExtractJoinFilterRule;
 import org.apache.hadoop.hive.ql.optimizer.calcite.rules.jdbc.JDBCFilterJoinRule;
 import org.apache.hadoop.hive.ql.optimizer.calcite.rules.jdbc.JDBCFilterPushDownRule;
-import org.apache.hadoop.hive.ql.optimizer.calcite.rules.jdbc.JDBCExtractJoinFilterRule;
 import org.apache.hadoop.hive.ql.optimizer.calcite.rules.jdbc.JDBCJoinPushDownRule;
 import org.apache.hadoop.hive.ql.optimizer.calcite.rules.jdbc.JDBCProjectPushDownRule;
 import org.apache.hadoop.hive.ql.optimizer.calcite.rules.jdbc.JDBCSortPushDownRule;
 import org.apache.hadoop.hive.ql.optimizer.calcite.rules.jdbc.JDBCUnionPushDownRule;
-import org.apache.hadoop.hive.ql.optimizer.calcite.rules.jdbc.JDBCAbstractSplitFilterRule;
-
 import org.apache.hadoop.hive.ql.optimizer.calcite.rules.views.HiveAggregateIncrementalRewritingRule;
 import org.apache.hadoop.hive.ql.optimizer.calcite.rules.views.HiveMaterializedViewRule;
 import org.apache.hadoop.hive.ql.optimizer.calcite.rules.views.HiveNoAggregateIncrementalRewritingRule;
 import org.apache.hadoop.hive.ql.optimizer.calcite.rules.views.MaterializedViewRewritingRelVisitor;
-
 import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ASTBuilder;
 import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ASTConverter;
 import org.apache.hadoop.hive.ql.optimizer.calcite.translator.HiveOpConverter;
@@ -295,13 +272,32 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
 import org.joda.time.Interval;
 
-import com.google.common.base.Function;
-import com.google.common.collect.ArrayListMultimap;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableList.Builder;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Multimap;
+import javax.sql.DataSource;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.UndeclaredThrowableException;
+import java.math.BigDecimal;
+import java.util.AbstractMap.SimpleEntry;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Deque;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 
 
 public class CalcitePlanner extends SemanticAnalyzer {
@@ -1833,20 +1829,20 @@ public class CalcitePlanner extends SemanticAnalyzer {
       // 11. Apply Druid transformation rules
       perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER);
       calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), null,
-              HepMatchOrder.BOTTOM_UP,
-              HiveDruidRules.FILTER,
-              HiveDruidRules.PROJECT_FILTER_TRANSPOSE,
-              HiveDruidRules.AGGREGATE_FILTER_TRANSPOSE,
-              HiveDruidRules.AGGREGATE_PROJECT,
-              HiveDruidRules.PROJECT,
-              HiveDruidRules.AGGREGATE,
-              HiveDruidRules.POST_AGGREGATION_PROJECT,
-              HiveDruidRules.FILTER_AGGREGATE_TRANSPOSE,
-              HiveDruidRules.FILTER_PROJECT_TRANSPOSE,
-              HiveDruidRules.HAVING_FILTER_RULE,
-              HiveDruidRules.SORT_PROJECT_TRANSPOSE,
-              HiveDruidRules.SORT,
-              HiveDruidRules.PROJECT_SORT_TRANSPOSE
+          HepMatchOrder.BOTTOM_UP,
+          HiveDruidRules.FILTER, HiveDruidRules.PROJECT_FILTER_TRANSPOSE,
+          HiveDruidRules.AGGREGATE_FILTER_TRANSPOSE,
+          HiveDruidRules.AGGREGATE_PROJECT,
+          HiveDruidRules.PROJECT,
+          HiveDruidRules.EXPAND_SINGLE_DISTINCT_AGGREGATES_DRUID_RULE,
+          HiveDruidRules.AGGREGATE,
+          HiveDruidRules.POST_AGGREGATION_PROJECT,
+          HiveDruidRules.FILTER_AGGREGATE_TRANSPOSE,
+          HiveDruidRules.FILTER_PROJECT_TRANSPOSE,
+          HiveDruidRules.HAVING_FILTER_RULE,
+          HiveDruidRules.SORT_PROJECT_TRANSPOSE,
+          HiveDruidRules.SORT,
+          HiveDruidRules.PROJECT_SORT_TRANSPOSE
       );
       perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Druid transformation rules");
 

http://git-wip-us.apache.org/repos/asf/hive/blob/7ed820f2/ql/src/test/queries/clientpositive/druidmini_expressions.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/druidmini_expressions.q b/ql/src/test/queries/clientpositive/druidmini_expressions.q
index 722936b..72e6256 100644
--- a/ql/src/test/queries/clientpositive/druidmini_expressions.q
+++ b/ql/src/test/queries/clientpositive/druidmini_expressions.q
@@ -51,4 +51,34 @@ EXPLAIN SELECT cstring1 || '_'|| cstring2, substring(cstring2, 2, 3) as concat ,
 
 explain extended select count(*) from (select `__time` from druid_table_n0 limit 1) as src ;
 
+SELECT `__time`
+FROM druid_table_n0
+WHERE (`__time` BETWEEN '1968-01-01 00:00:00' AND '1970-01-01 00:00:00')
+    OR (`__time` BETWEEN '1968-02-01 00:00:00' AND '1970-04-01 00:00:00') ORDER BY `__time` ASC LIMIT 10;
+
+-- COUNT DISTINCT TESTS
+-- AS PART OF https://issues.apache.org/jira/browse/HIVE-19586
+
+EXPLAIN select count(DISTINCT cstring2), sum(cdouble) FROM druid_table_n0 GROUP  BY `__time`, `cstring1` ;
+
+EXPLAIN select count(distinct cdouble), sum(cdouble) FROM druid_table_n0 GROUP  BY `__time`, `cstring1` ;
+
+EXPLAIN select count(distinct cstring2), sum(2 * cdouble) FROM druid_table_n0 GROUP  BY `__time`, `cstring1` ;
+
+EXPLAIN select count(distinct cstring2 || '_'|| cstring1), sum(cdouble) FROM druid_table_n0 GROUP  BY `__time`, `cstring1` ;
+
+EXPLAIN select count(DISTINCT cstring2) FROM druid_table_n0 ;
+EXPLAIN select count(DISTINCT cstring2), sum(cdouble) FROM druid_table_n0 ;
+EXPLAIN select count(distinct cstring2 || '_'|| cstring1), sum(cdouble), min(cint) FROM druid_table_n0;
+
+select count(DISTINCT cstring2), sum(cdouble) FROM druid_table_n0 GROUP  BY floor_year(`__time`) ;
+
+select count(distinct cstring2), sum(2 * cdouble) FROM druid_table_n0 GROUP  BY floor_year(`__time`) ;
+
+select count(DISTINCT cstring2) FROM druid_table_n0 ;
+
+select count(DISTINCT cstring2), sum(cdouble) FROM druid_table_n0 ;
+
+select count(distinct cstring2 || '_'|| cstring1), sum(cdouble), min(cint) FROM druid_table_n0;
+
 DROP TABLE druid_table_n0;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/7ed820f2/ql/src/test/results/clientpositive/druid/druidmini_expressions.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/druid/druidmini_expressions.q.out b/ql/src/test/results/clientpositive/druid/druidmini_expressions.q.out
index d51c5af..30bc5d0 100644
--- a/ql/src/test/results/clientpositive/druid/druidmini_expressions.q.out
+++ b/ql/src/test/results/clientpositive/druid/druidmini_expressions.q.out
@@ -368,6 +368,527 @@ STAGE PLANS:
       Processor Tree:
         ListSink
 
+PREHOOK: query: SELECT `__time`
+FROM druid_table_n0
+WHERE (`__time` BETWEEN '1968-01-01 00:00:00' AND '1970-01-01 00:00:00')
+    OR (`__time` BETWEEN '1968-02-01 00:00:00' AND '1970-04-01 00:00:00') ORDER BY `__time` ASC LIMIT 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@druid_table_n0
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT `__time`
+FROM druid_table_n0
+WHERE (`__time` BETWEEN '1968-01-01 00:00:00' AND '1970-01-01 00:00:00')
+    OR (`__time` BETWEEN '1968-02-01 00:00:00' AND '1970-04-01 00:00:00') ORDER BY `__time` ASC LIMIT 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@druid_table_n0
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1969-12-31 15:59:00.0 US/Pacific
+1969-12-31 15:59:00.0 US/Pacific
+1969-12-31 15:59:00.0 US/Pacific
+1969-12-31 15:59:00.0 US/Pacific
+1969-12-31 15:59:00.0 US/Pacific
+1969-12-31 15:59:00.0 US/Pacific
+1969-12-31 15:59:00.0 US/Pacific
+1969-12-31 15:59:00.0 US/Pacific
+1969-12-31 15:59:00.0 US/Pacific
+1969-12-31 15:59:00.0 US/Pacific
+PREHOOK: query: EXPLAIN select count(DISTINCT cstring2), sum(cdouble) FROM druid_table_n0 GROUP  BY `__time`, `cstring1`
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN select count(DISTINCT cstring2), sum(cdouble) FROM druid_table_n0 GROUP  BY `__time`, `cstring1`
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Tez
+#### A masked pattern was here ####
+      Edges:
+        Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: druid_table_n0
+                  properties:
+                    druid.fieldNames extract,cstring1,cstring2,$f3
+                    druid.fieldTypes timestamp with local time zone,string,string,double
+                    druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_n0","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","timeZone":"US/Pacific"}},{"type":"default","dimension":"cstring1","outputName":"cstring1","outputType":"STRING"},{"type":"default","dimension":"cstring2","outputName":"cstring2","outputType":"STRING"}],"limitSpec":{"type":"default"},"aggregations":[{"type":"doubleSum","name":"$f3","fieldName":"cdouble"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]}
+                    druid.query.type groupBy
+                  Statistics: Num rows: 9173 Data size: 3625856 Basic stats: COMPLETE Column stats: NONE
+                  Select Operator
+                    expressions: extract (type: timestamp with local time zone), cstring1 (type: string), cstring2 (type: string), $f3 (type: double)
+                    outputColumnNames: extract, cstring1, cstring2, $f3
+                    Statistics: Num rows: 9173 Data size: 3625856 Basic stats: COMPLETE Column stats: NONE
+                    Group By Operator
+                      aggregations: count(cstring2), sum($f3)
+                      keys: extract (type: timestamp with local time zone), cstring1 (type: string)
+                      mode: hash
+                      outputColumnNames: _col0, _col1, _col2, _col3
+                      Statistics: Num rows: 9173 Data size: 3625856 Basic stats: COMPLETE Column stats: NONE
+                      Reduce Output Operator
+                        key expressions: _col0 (type: timestamp with local time zone), _col1 (type: string)
+                        sort order: ++
+                        Map-reduce partition columns: _col0 (type: timestamp with local time zone), _col1 (type: string)
+                        Statistics: Num rows: 9173 Data size: 3625856 Basic stats: COMPLETE Column stats: NONE
+                        value expressions: _col2 (type: bigint), _col3 (type: double)
+        Reducer 2 
+            Reduce Operator Tree:
+              Group By Operator
+                aggregations: count(VALUE._col0), sum(VALUE._col1)
+                keys: KEY._col0 (type: timestamp with local time zone), KEY._col1 (type: string)
+                mode: mergepartial
+                outputColumnNames: _col0, _col1, _col2, _col3
+                Statistics: Num rows: 4586 Data size: 1812730 Basic stats: COMPLETE Column stats: NONE
+                Select Operator
+                  expressions: _col2 (type: bigint), _col3 (type: double)
+                  outputColumnNames: _col0, _col1
+                  Statistics: Num rows: 4586 Data size: 1812730 Basic stats: COMPLETE Column stats: NONE
+                  File Output Operator
+                    compressed: false
+                    Statistics: Num rows: 4586 Data size: 1812730 Basic stats: COMPLETE Column stats: NONE
+                    table:
+                        input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                        output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+PREHOOK: query: EXPLAIN select count(distinct cdouble), sum(cdouble) FROM druid_table_n0 GROUP  BY `__time`, `cstring1`
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN select count(distinct cdouble), sum(cdouble) FROM druid_table_n0 GROUP  BY `__time`, `cstring1`
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Tez
+#### A masked pattern was here ####
+      Edges:
+        Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: druid_table_n0
+                  properties:
+                    druid.fieldNames extract,cstring1,cdouble,$f3
+                    druid.fieldTypes timestamp with local time zone,string,double,double
+                    druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_n0","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","timeZone":"US/Pacific"}},{"type":"default","dimension":"cstring1","outputName":"cstring1","outputType":"STRING"},{"type":"default","dimension":"cdouble","outputName":"cdouble","outputType":"DOUBLE"}],"limitSpec":{"type":"default"},"aggregations":[{"type":"doubleSum","name":"$f3","fieldName":"cdouble"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]}
+                    druid.query.type groupBy
+                  Statistics: Num rows: 9173 Data size: 2091840 Basic stats: COMPLETE Column stats: NONE
+                  Select Operator
+                    expressions: extract (type: timestamp with local time zone), cstring1 (type: string), cdouble (type: double), $f3 (type: double)
+                    outputColumnNames: extract, cstring1, cdouble, $f3
+                    Statistics: Num rows: 9173 Data size: 2091840 Basic stats: COMPLETE Column stats: NONE
+                    Group By Operator
+                      aggregations: count(cdouble), sum($f3)
+                      keys: extract (type: timestamp with local time zone), cstring1 (type: string)
+                      mode: hash
+                      outputColumnNames: _col0, _col1, _col2, _col3
+                      Statistics: Num rows: 9173 Data size: 2091840 Basic stats: COMPLETE Column stats: NONE
+                      Reduce Output Operator
+                        key expressions: _col0 (type: timestamp with local time zone), _col1 (type: string)
+                        sort order: ++
+                        Map-reduce partition columns: _col0 (type: timestamp with local time zone), _col1 (type: string)
+                        Statistics: Num rows: 9173 Data size: 2091840 Basic stats: COMPLETE Column stats: NONE
+                        value expressions: _col2 (type: bigint), _col3 (type: double)
+        Reducer 2 
+            Reduce Operator Tree:
+              Group By Operator
+                aggregations: count(VALUE._col0), sum(VALUE._col1)
+                keys: KEY._col0 (type: timestamp with local time zone), KEY._col1 (type: string)
+                mode: mergepartial
+                outputColumnNames: _col0, _col1, _col2, _col3
+                Statistics: Num rows: 4586 Data size: 1045805 Basic stats: COMPLETE Column stats: NONE
+                Select Operator
+                  expressions: _col2 (type: bigint), _col3 (type: double)
+                  outputColumnNames: _col0, _col1
+                  Statistics: Num rows: 4586 Data size: 1045805 Basic stats: COMPLETE Column stats: NONE
+                  File Output Operator
+                    compressed: false
+                    Statistics: Num rows: 4586 Data size: 1045805 Basic stats: COMPLETE Column stats: NONE
+                    table:
+                        input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                        output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+PREHOOK: query: EXPLAIN select count(distinct cstring2), sum(2 * cdouble) FROM druid_table_n0 GROUP  BY `__time`, `cstring1`
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN select count(distinct cstring2), sum(2 * cdouble) FROM druid_table_n0 GROUP  BY `__time`, `cstring1`
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Tez
+#### A masked pattern was here ####
+      Edges:
+        Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: druid_table_n0
+                  properties:
+                    druid.fieldNames extract,cstring1,cstring2,$f3
+                    druid.fieldTypes timestamp with local time zone,string,string,double
+                    druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_n0","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","timeZone":"US/Pacific"}},{"type":"default","dimension":"cstring1","outputName":"cstring1","outputType":"STRING"},{"type":"default","dimension":"cstring2","outputName":"cstring2","outputType":"STRING"}],"limitSpec":{"type":"default"},"aggregations":[{"type":"doubleSum","name":"$f3","expression":"(CAST(2, 'DOUBLE') * \"cdouble\")"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]}
+                    druid.query.type groupBy
+                  Statistics: Num rows: 9173 Data size: 3625856 Basic stats: COMPLETE Column stats: NONE
+                  Select Operator
+                    expressions: extract (type: timestamp with local time zone), cstring1 (type: string), cstring2 (type: string), $f3 (type: double)
+                    outputColumnNames: extract, cstring1, cstring2, $f3
+                    Statistics: Num rows: 9173 Data size: 3625856 Basic stats: COMPLETE Column stats: NONE
+                    Group By Operator
+                      aggregations: count(cstring2), sum($f3)
+                      keys: extract (type: timestamp with local time zone), cstring1 (type: string)
+                      mode: hash
+                      outputColumnNames: _col0, _col1, _col2, _col3
+                      Statistics: Num rows: 9173 Data size: 3625856 Basic stats: COMPLETE Column stats: NONE
+                      Reduce Output Operator
+                        key expressions: _col0 (type: timestamp with local time zone), _col1 (type: string)
+                        sort order: ++
+                        Map-reduce partition columns: _col0 (type: timestamp with local time zone), _col1 (type: string)
+                        Statistics: Num rows: 9173 Data size: 3625856 Basic stats: COMPLETE Column stats: NONE
+                        value expressions: _col2 (type: bigint), _col3 (type: double)
+        Reducer 2 
+            Reduce Operator Tree:
+              Group By Operator
+                aggregations: count(VALUE._col0), sum(VALUE._col1)
+                keys: KEY._col0 (type: timestamp with local time zone), KEY._col1 (type: string)
+                mode: mergepartial
+                outputColumnNames: _col0, _col1, _col2, _col3
+                Statistics: Num rows: 4586 Data size: 1812730 Basic stats: COMPLETE Column stats: NONE
+                Select Operator
+                  expressions: _col2 (type: bigint), _col3 (type: double)
+                  outputColumnNames: _col0, _col1
+                  Statistics: Num rows: 4586 Data size: 1812730 Basic stats: COMPLETE Column stats: NONE
+                  File Output Operator
+                    compressed: false
+                    Statistics: Num rows: 4586 Data size: 1812730 Basic stats: COMPLETE Column stats: NONE
+                    table:
+                        input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                        output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+PREHOOK: query: EXPLAIN select count(distinct cstring2 || '_'|| cstring1), sum(cdouble) FROM druid_table_n0 GROUP  BY `__time`, `cstring1`
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN select count(distinct cstring2 || '_'|| cstring1), sum(cdouble) FROM druid_table_n0 GROUP  BY `__time`, `cstring1`
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Tez
+#### A masked pattern was here ####
+      Edges:
+        Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: druid_table_n0
+                  properties:
+                    druid.fieldNames extract,cstring1,vc,$f3
+                    druid.fieldTypes timestamp with local time zone,string,string,double
+                    druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_n0","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","timeZone":"US/Pacific"}},{"type":"default","dimension":"cstring1","outputName":"cstring1","outputType":"STRING"},{"type":"default","dimension":"vc","outputName":"vc","outputType":"STRING"}],"virtualColumns":[{"type":"expression","name":"vc","expression":"concat(concat(\"cstring2\",'_'),\"cstring1\")","outputType":"STRING"}],"limitSpec":{"type":"default"},"aggregations":[{"type":"doubleSum","name":"$f3","fieldName":"cdouble"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]}
+                    druid.query.type groupBy
+                  Statistics: Num rows: 9173 Data size: 3625856 Basic stats: COMPLETE Column stats: NONE
+                  Select Operator
+                    expressions: extract (type: timestamp with local time zone), cstring1 (type: string), vc (type: string), $f3 (type: double)
+                    outputColumnNames: extract, cstring1, vc, $f3
+                    Statistics: Num rows: 9173 Data size: 3625856 Basic stats: COMPLETE Column stats: NONE
+                    Group By Operator
+                      aggregations: count(vc), sum($f3)
+                      keys: extract (type: timestamp with local time zone), cstring1 (type: string)
+                      mode: hash
+                      outputColumnNames: _col0, _col1, _col2, _col3
+                      Statistics: Num rows: 9173 Data size: 3625856 Basic stats: COMPLETE Column stats: NONE
+                      Reduce Output Operator
+                        key expressions: _col0 (type: timestamp with local time zone), _col1 (type: string)
+                        sort order: ++
+                        Map-reduce partition columns: _col0 (type: timestamp with local time zone), _col1 (type: string)
+                        Statistics: Num rows: 9173 Data size: 3625856 Basic stats: COMPLETE Column stats: NONE
+                        value expressions: _col2 (type: bigint), _col3 (type: double)
+        Reducer 2 
+            Reduce Operator Tree:
+              Group By Operator
+                aggregations: count(VALUE._col0), sum(VALUE._col1)
+                keys: KEY._col0 (type: timestamp with local time zone), KEY._col1 (type: string)
+                mode: mergepartial
+                outputColumnNames: _col0, _col1, _col2, _col3
+                Statistics: Num rows: 4586 Data size: 1812730 Basic stats: COMPLETE Column stats: NONE
+                Select Operator
+                  expressions: _col2 (type: bigint), _col3 (type: double)
+                  outputColumnNames: _col0, _col1
+                  Statistics: Num rows: 4586 Data size: 1812730 Basic stats: COMPLETE Column stats: NONE
+                  File Output Operator
+                    compressed: false
+                    Statistics: Num rows: 4586 Data size: 1812730 Basic stats: COMPLETE Column stats: NONE
+                    table:
+                        input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                        output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+PREHOOK: query: EXPLAIN select count(DISTINCT cstring2) FROM druid_table_n0
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN select count(DISTINCT cstring2) FROM druid_table_n0
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Tez
+#### A masked pattern was here ####
+      Edges:
+        Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: druid_table_n0
+                  properties:
+                    druid.fieldNames cstring2
+                    druid.fieldTypes string
+                    druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_n0","granularity":"all","dimensions":[{"type":"default","dimension":"cstring2","outputName":"cstring2","outputType":"STRING"}],"limitSpec":{"type":"default"},"aggregations":[],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]}
+                    druid.query.type groupBy
+                  Statistics: Num rows: 9173 Data size: 1603744 Basic stats: COMPLETE Column stats: NONE
+                  Select Operator
+                    expressions: cstring2 (type: string)
+                    outputColumnNames: cstring2
+                    Statistics: Num rows: 9173 Data size: 1603744 Basic stats: COMPLETE Column stats: NONE
+                    Group By Operator
+                      aggregations: count(cstring2)
+                      mode: hash
+                      outputColumnNames: _col0
+                      Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE
+                      Reduce Output Operator
+                        sort order: 
+                        Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE
+                        value expressions: _col0 (type: bigint)
+        Reducer 2 
+            Reduce Operator Tree:
+              Group By Operator
+                aggregations: count(VALUE._col0)
+                mode: mergepartial
+                outputColumnNames: _col0
+                Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+PREHOOK: query: EXPLAIN select count(DISTINCT cstring2), sum(cdouble) FROM druid_table_n0
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN select count(DISTINCT cstring2), sum(cdouble) FROM druid_table_n0
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Tez
+#### A masked pattern was here ####
+      Edges:
+        Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: druid_table_n0
+                  properties:
+                    druid.fieldNames cstring2,$f1
+                    druid.fieldTypes string,double
+                    druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_n0","granularity":"all","dimensions":[{"type":"default","dimension":"cstring2","outputName":"cstring2","outputType":"STRING"}],"limitSpec":{"type":"default"},"aggregations":[{"type":"doubleSum","name":"$f1","fieldName":"cdouble"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]}
+                    druid.query.type groupBy
+                  Statistics: Num rows: 9173 Data size: 1673472 Basic stats: COMPLETE Column stats: NONE
+                  Select Operator
+                    expressions: cstring2 (type: string), $f1 (type: double)
+                    outputColumnNames: cstring2, $f1
+                    Statistics: Num rows: 9173 Data size: 1673472 Basic stats: COMPLETE Column stats: NONE
+                    Group By Operator
+                      aggregations: count(cstring2), sum($f1)
+                      mode: hash
+                      outputColumnNames: _col0, _col1
+                      Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE
+                      Reduce Output Operator
+                        sort order: 
+                        Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE
+                        value expressions: _col0 (type: bigint), _col1 (type: double)
+        Reducer 2 
+            Reduce Operator Tree:
+              Group By Operator
+                aggregations: count(VALUE._col0), sum(VALUE._col1)
+                mode: mergepartial
+                outputColumnNames: _col0, _col1
+                Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+PREHOOK: query: EXPLAIN select count(distinct cstring2 || '_'|| cstring1), sum(cdouble), min(cint) FROM druid_table_n0
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN select count(distinct cstring2 || '_'|| cstring1), sum(cdouble), min(cint) FROM druid_table_n0
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Tez
+#### A masked pattern was here ####
+      Edges:
+        Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: druid_table_n0
+                  properties:
+                    druid.fieldNames vc,$f1,$f2
+                    druid.fieldTypes string,double,int
+                    druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_n0","granularity":"all","dimensions":[{"type":"default","dimension":"vc","outputName":"vc","outputType":"STRING"}],"virtualColumns":[{"type":"expression","name":"vc","expression":"concat(concat(\"cstring2\",'_'),\"cstring1\")","outputType":"STRING"}],"limitSpec":{"type":"default"},"aggregations":[{"type":"doubleSum","name":"$f1","fieldName":"cdouble"},{"type":"longMin","name":"$f2","fieldName":"cint"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]}
+                    druid.query.type groupBy
+                  Statistics: Num rows: 9173 Data size: 1708336 Basic stats: COMPLETE Column stats: NONE
+                  Select Operator
+                    expressions: vc (type: string), $f1 (type: double), $f2 (type: int)
+                    outputColumnNames: vc, $f1, $f2
+                    Statistics: Num rows: 9173 Data size: 1708336 Basic stats: COMPLETE Column stats: NONE
+                    Group By Operator
+                      aggregations: count(vc), sum($f1), min($f2)
+                      mode: hash
+                      outputColumnNames: _col0, _col1, _col2
+                      Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE
+                      Reduce Output Operator
+                        sort order: 
+                        Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE
+                        value expressions: _col0 (type: bigint), _col1 (type: double), _col2 (type: int)
+        Reducer 2 
+            Reduce Operator Tree:
+              Group By Operator
+                aggregations: count(VALUE._col0), sum(VALUE._col1), min(VALUE._col2)
+                mode: mergepartial
+                outputColumnNames: _col0, _col1, _col2
+                Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+PREHOOK: query: select count(DISTINCT cstring2), sum(cdouble) FROM druid_table_n0 GROUP  BY floor_year(`__time`)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@druid_table_n0
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(DISTINCT cstring2), sum(cdouble) FROM druid_table_n0 GROUP  BY floor_year(`__time`)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@druid_table_n0
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+6078	2.7308662809692383E7
+PREHOOK: query: select count(distinct cstring2), sum(2 * cdouble) FROM druid_table_n0 GROUP  BY floor_year(`__time`)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@druid_table_n0
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(distinct cstring2), sum(2 * cdouble) FROM druid_table_n0 GROUP  BY floor_year(`__time`)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@druid_table_n0
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+6078	5.4617325619384766E7
+PREHOOK: query: select count(DISTINCT cstring2) FROM druid_table_n0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@druid_table_n0
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(DISTINCT cstring2) FROM druid_table_n0
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@druid_table_n0
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+6078
+PREHOOK: query: select count(DISTINCT cstring2), sum(cdouble) FROM druid_table_n0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@druid_table_n0
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(DISTINCT cstring2), sum(cdouble) FROM druid_table_n0
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@druid_table_n0
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+6078	2.7308662809692383E7
+PREHOOK: query: select count(distinct cstring2 || '_'|| cstring1), sum(cdouble), min(cint) FROM druid_table_n0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@druid_table_n0
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(distinct cstring2 || '_'|| cstring1), sum(cdouble), min(cint) FROM druid_table_n0
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@druid_table_n0
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+6095	2.7308662809692383E7	-1073279343
 PREHOOK: query: DROP TABLE druid_table_n0
 PREHOOK: type: DROPTABLE
 PREHOOK: Input: default@druid_table_n0


Mime
View raw message