Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id EAF2B200BA6 for ; Tue, 18 Oct 2016 13:28:40 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id E9AB9160ACC; Tue, 18 Oct 2016 11:28:40 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 94C73160ADC for ; Tue, 18 Oct 2016 13:28:38 +0200 (CEST) Received: (qmail 30875 invoked by uid 500); 18 Oct 2016 11:28:37 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 30836 invoked by uid 99); 18 Oct 2016 11:28:37 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 18 Oct 2016 11:28:37 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 40D28DFB81; Tue, 18 Oct 2016 11:28:37 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jcamacho@apache.org To: commits@hive.apache.org Date: Tue, 18 Oct 2016 11:28:42 -0000 Message-Id: <469f4c1be05e493589db3de5c617e366@git.apache.org> In-Reply-To: <18bf3838213947f3bdc0324291f1a768@git.apache.org> References: <18bf3838213947f3bdc0324291f1a768@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [6/6] hive git commit: HIVE-13316: Upgrade to Calcite 1.10 (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan) archived-at: Tue, 18 Oct 2016 11:28:41 -0000 HIVE-13316: Upgrade to Calcite 1.10 (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b597ab2a Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b597ab2a Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b597ab2a Branch: refs/heads/master Commit: b597ab2a07034b9c82e4bb0591123c3a115f27eb Parents: 4b7f373 Author: Jesus Camacho Rodriguez Authored: Wed Sep 28 20:23:33 2016 +0100 Committer: Jesus Camacho Rodriguez Committed: Tue Oct 18 12:27:41 2016 +0100 ---------------------------------------------------------------------- .../druid/HiveDruidQueryBasedInputFormat.java | 6 +- .../serde/DruidGroupByQueryRecordReader.java | 2 +- .../serde/DruidSelectQueryRecordReader.java | 2 +- .../hadoop/hive/druid/serde/DruidSerDe.java | 2 +- .../serde/DruidTimeseriesQueryRecordReader.java | 2 +- .../druid/serde/DruidTopNQueryRecordReader.java | 2 +- pom.xml | 3 +- ql/pom.xml | 7 +- .../calcite/HiveDefaultRelMetadataProvider.java | 2 +- .../optimizer/calcite/HivePlannerContext.java | 9 +- .../ql/optimizer/calcite/HiveRelBuilder.java | 18 +- .../ql/optimizer/calcite/HiveRelOptUtil.java | 8 +- .../hive/ql/optimizer/calcite/HiveRexUtil.java | 821 -------------- .../optimizer/calcite/HiveTypeSystemImpl.java | 39 +- .../calcite/cost/HiveDefaultCostModel.java | 7 +- .../optimizer/calcite/cost/HiveRelMdCost.java | 10 +- .../calcite/druid/DruidIntervalUtils.java | 466 -------- .../ql/optimizer/calcite/druid/DruidQuery.java | 1053 ------------------ .../optimizer/calcite/druid/DruidQueryType.java | 42 - .../ql/optimizer/calcite/druid/DruidRules.java | 591 ---------- .../ql/optimizer/calcite/druid/DruidSchema.java | 51 - .../ql/optimizer/calcite/druid/DruidTable.java | 121 -- .../optimizer/calcite/druid/HiveDruidConf.java | 33 - .../calcite/reloperators/HiveAggregate.java | 3 +- .../reloperators/HiveDateGranularity.java | 54 - .../calcite/reloperators/HiveExtractDate.java | 50 + .../calcite/reloperators/HiveFloorDate.java | 64 ++ .../rules/HiveAggregateJoinTransposeRule.java | 9 +- .../rules/HiveAggregateProjectMergeRule.java | 3 +- .../rules/HiveFilterProjectTSTransposeRule.java | 16 +- .../rules/HiveFilterProjectTransposeRule.java | 21 +- .../calcite/rules/HivePreFilteringRule.java | 7 +- .../rules/HiveReduceExpressionsRule.java | 914 ++------------- .../HiveReduceExpressionsWithStatsRule.java | 5 +- .../calcite/rules/HiveRelFieldTrimmer.java | 243 +--- .../calcite/stats/HiveRelMdCollation.java | 10 +- .../calcite/stats/HiveRelMdDistribution.java | 10 +- .../calcite/stats/HiveRelMdPredicates.java | 31 +- .../calcite/stats/HiveRelMdSelectivity.java | 28 +- .../optimizer/calcite/stats/HiveRelMdSize.java | 13 +- .../calcite/stats/HiveRelMdUniqueKeys.java | 72 +- .../calcite/translator/ASTBuilder.java | 49 +- .../calcite/translator/ASTConverter.java | 51 +- .../calcite/translator/ExprNodeConverter.java | 49 +- .../translator/PlanModifierForASTConv.java | 5 + .../calcite/translator/RexNodeConverter.java | 61 +- .../translator/SqlFunctionConverter.java | 37 +- .../calcite/translator/TypeConverter.java | 41 +- .../hadoop/hive/ql/parse/CalcitePlanner.java | 40 +- .../hive/ql/parse/TypeCheckProcFactory.java | 2 +- .../optimizer/calcite/TestCBOMaxNumToCNF.java | 5 +- .../calcite/TestCBORuleFiredOnlyOnce.java | 2 +- .../results/clientpositive/druid_basic2.q.out | 48 +- .../clientpositive/druid_intervals.q.out | 40 +- .../clientpositive/druid_timeseries.q.out | 52 +- .../results/clientpositive/druid_topn.q.out | 32 +- .../clientpositive/explain_logical.q.out | 48 +- .../clientpositive/groupby_sort_1_23.q.out | 40 +- .../clientpositive/groupby_sort_skew_1_23.q.out | 40 +- .../results/clientpositive/limit_pushdown.q.out | 12 +- .../clientpositive/limit_pushdown3.q.out | 12 +- .../clientpositive/llap/explainuser_4.q.out | 32 +- .../clientpositive/llap/limit_pushdown.q.out | 9 +- .../results/clientpositive/llap/lineage3.q.out | 2 +- .../llap/table_access_keys_stats.q.out | 6 +- .../llap/tez_dynpart_hashjoin_1.q.out | 42 +- .../llap/tez_vector_dynpart_hashjoin_1.q.out | 42 +- .../offset_limit_ppd_optimizer.q.out | 12 +- .../results/clientpositive/perf/query75.q.out | 12 +- .../spark/groupby_sort_1_23.q.out | 32 +- .../spark/groupby_sort_skew_1_23.q.out | 32 +- .../clientpositive/spark/limit_pushdown.q.out | 9 +- .../spark/table_access_keys_stats.q.out | 6 +- .../clientpositive/tez/explainanalyze_4.q.out | 32 +- .../tez/vectorization_limit.q.out | 9 +- .../clientpositive/vectorization_limit.q.out | 12 +- 76 files changed, 1136 insertions(+), 4669 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/b597ab2a/druid-handler/src/java/org/apache/hadoop/hive/druid/HiveDruidQueryBasedInputFormat.java ---------------------------------------------------------------------- diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/HiveDruidQueryBasedInputFormat.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/HiveDruidQueryBasedInputFormat.java index 3df1452..a18e590 100644 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/HiveDruidQueryBasedInputFormat.java +++ b/druid-handler/src/java/org/apache/hadoop/hive/druid/HiveDruidQueryBasedInputFormat.java @@ -25,6 +25,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.calcite.adapter.druid.DruidDateTimeUtils; +import org.apache.calcite.adapter.druid.DruidTable; import org.apache.commons.lang3.StringEscapeUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; @@ -37,8 +39,6 @@ import org.apache.hadoop.hive.druid.serde.DruidSelectQueryRecordReader; import org.apache.hadoop.hive.druid.serde.DruidTimeseriesQueryRecordReader; import org.apache.hadoop.hive.druid.serde.DruidTopNQueryRecordReader; import org.apache.hadoop.hive.druid.serde.DruidWritable; -import org.apache.hadoop.hive.ql.optimizer.calcite.druid.DruidIntervalUtils; -import org.apache.hadoop.hive.ql.optimizer.calcite.druid.DruidTable; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapred.JobConf; @@ -273,7 +273,7 @@ public class HiveDruidQueryBasedInputFormat extends InputFormat> createSplitsIntervals(List intervals, int numSplits) { - final long totalTime = DruidIntervalUtils.extractTotalTime(intervals); + final long totalTime = DruidDateTimeUtils.extractTotalTime(intervals); long startTime = intervals.get(0).getStartMillis(); long endTime = startTime; long currTime = 0; http://git-wip-us.apache.org/repos/asf/hive/blob/b597ab2a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidGroupByQueryRecordReader.java ---------------------------------------------------------------------- diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidGroupByQueryRecordReader.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidGroupByQueryRecordReader.java index 226060f..49e096b 100644 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidGroupByQueryRecordReader.java +++ b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidGroupByQueryRecordReader.java @@ -21,9 +21,9 @@ import java.io.IOException; import java.io.InputStream; import java.util.List; +import org.apache.calcite.adapter.druid.DruidTable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.druid.DruidStorageHandlerUtils; -import org.apache.hadoop.hive.ql.optimizer.calcite.druid.DruidTable; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.InputSplit; http://git-wip-us.apache.org/repos/asf/hive/blob/b597ab2a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSelectQueryRecordReader.java ---------------------------------------------------------------------- diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSelectQueryRecordReader.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSelectQueryRecordReader.java index 70b493c..fccf7c4 100644 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSelectQueryRecordReader.java +++ b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSelectQueryRecordReader.java @@ -22,8 +22,8 @@ import java.io.InputStream; import java.util.Iterator; import java.util.List; +import org.apache.calcite.adapter.druid.DruidTable; import org.apache.hadoop.hive.druid.DruidStorageHandlerUtils; -import org.apache.hadoop.hive.ql.optimizer.calcite.druid.DruidTable; import org.apache.hadoop.io.NullWritable; import com.fasterxml.jackson.core.type.TypeReference; http://git-wip-us.apache.org/repos/asf/hive/blob/b597ab2a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java ---------------------------------------------------------------------- diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java index 8f53d4a..238f7a3 100644 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java +++ b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java @@ -25,11 +25,11 @@ import java.util.List; import java.util.Map.Entry; import java.util.Properties; +import org.apache.calcite.adapter.druid.DruidTable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.druid.DruidStorageHandlerUtils; -import org.apache.hadoop.hive.ql.optimizer.calcite.druid.DruidTable; import org.apache.hadoop.hive.serde2.AbstractSerDe; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.SerDeSpec; http://git-wip-us.apache.org/repos/asf/hive/blob/b597ab2a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidTimeseriesQueryRecordReader.java ---------------------------------------------------------------------- diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidTimeseriesQueryRecordReader.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidTimeseriesQueryRecordReader.java index 812ae03..b91178c 100644 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidTimeseriesQueryRecordReader.java +++ b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidTimeseriesQueryRecordReader.java @@ -21,8 +21,8 @@ import java.io.IOException; import java.io.InputStream; import java.util.List; +import org.apache.calcite.adapter.druid.DruidTable; import org.apache.hadoop.hive.druid.DruidStorageHandlerUtils; -import org.apache.hadoop.hive.ql.optimizer.calcite.druid.DruidTable; import org.apache.hadoop.io.NullWritable; import com.fasterxml.jackson.core.type.TypeReference; http://git-wip-us.apache.org/repos/asf/hive/blob/b597ab2a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidTopNQueryRecordReader.java ---------------------------------------------------------------------- diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidTopNQueryRecordReader.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidTopNQueryRecordReader.java index 0b87976..0b77a9b 100644 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidTopNQueryRecordReader.java +++ b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidTopNQueryRecordReader.java @@ -22,8 +22,8 @@ import java.io.InputStream; import java.util.Iterator; import java.util.List; +import org.apache.calcite.adapter.druid.DruidTable; import org.apache.hadoop.hive.druid.DruidStorageHandlerUtils; -import org.apache.hadoop.hive.ql.optimizer.calcite.druid.DruidTable; import org.apache.hadoop.io.NullWritable; import com.fasterxml.jackson.core.type.TypeReference; http://git-wip-us.apache.org/repos/asf/hive/blob/b597ab2a/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 5d13344..98d2dc2 100644 --- a/pom.xml +++ b/pom.xml @@ -112,9 +112,10 @@ 3.4 1.5.6 0.1 + 1.8.0 1.7.7 0.8.0.RELEASE - 1.6.0 + 1.10.0 4.2.1 4.1.6 4.1.7 http://git-wip-us.apache.org/repos/asf/hive/blob/b597ab2a/ql/pom.xml ---------------------------------------------------------------------- diff --git a/ql/pom.xml b/ql/pom.xml index 2a93bb7..489c6f3 100644 --- a/ql/pom.xml +++ b/ql/pom.xml @@ -383,8 +383,13 @@ org.apache.calcite - calcite-avatica + calcite-druid ${calcite.version} + + + org.apache.calcite.avatica + avatica + ${avatica.version}