Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 49EFB200C8C for ; Tue, 6 Jun 2017 20:35:31 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 4860D160BEB; Tue, 6 Jun 2017 18:35:31 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 0C972160BD4 for ; Tue, 6 Jun 2017 20:35:29 +0200 (CEST) Received: (qmail 60292 invoked by uid 500); 6 Jun 2017 18:35:28 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 60272 invoked by uid 99); 6 Jun 2017 18:35:28 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 06 Jun 2017 18:35:28 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 8494CDFF81; Tue, 6 Jun 2017 18:35:28 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: weiz@apache.org To: commits@hive.apache.org Date: Tue, 06 Jun 2017 18:35:28 -0000 Message-Id: <42123c3b111445eb894fa8e052bae506@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [01/36] hive git commit: HIVE-16757 Remove calls to deprecated AbstractRelNode.getRows (Remus Rusanu reviewed by Ashutosh Chauhan) archived-at: Tue, 06 Jun 2017 18:35:31 -0000 Repository: hive Updated Branches: refs/heads/hive-14535 d9e4ae09e -> 78f28c885 HIVE-16757 Remove calls to deprecated AbstractRelNode.getRows (Remus Rusanu reviewed by Ashutosh Chauhan) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8aee8d4f Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8aee8d4f Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8aee8d4f Branch: refs/heads/hive-14535 Commit: 8aee8d4f2b124fcfa093724b4de0a54287a8084f Parents: 86b1877 Author: Remus Rusanu Authored: Tue May 30 22:37:54 2017 -0700 Committer: Remus Rusanu Committed: Tue May 30 22:37:54 2017 -0700 ---------------------------------------------------------------------- .../hive/ql/optimizer/calcite/cost/HiveAlgorithmsUtil.java | 4 ++-- .../hive/ql/optimizer/calcite/cost/HiveCostModel.java | 7 ++++--- .../ql/optimizer/calcite/cost/HiveDefaultCostModel.java | 2 +- .../hive/ql/optimizer/calcite/cost/HiveOnTezCostModel.java | 4 ++-- .../hive/ql/optimizer/calcite/cost/HiveRelMdCost.java | 2 +- .../optimizer/calcite/stats/FilterSelectivityEstimator.java | 9 +++++---- .../optimizer/calcite/stats/HiveRelMdDistinctRowCount.java | 2 +- .../hive/ql/optimizer/calcite/stats/HiveRelMdRowCount.java | 3 ++- .../ql/optimizer/calcite/stats/HiveRelMdSelectivity.java | 4 ++-- .../ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java | 2 +- 10 files changed, 21 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveAlgorithmsUtil.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveAlgorithmsUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveAlgorithmsUtil.java index 4ba10e2..28c7362 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveAlgorithmsUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveAlgorithmsUtil.java @@ -60,8 +60,8 @@ public class HiveAlgorithmsUtil { * Double.parseDouble(HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_CBO_COST_MODEL_HDFS_READ)); } - public static RelOptCost computeCardinalityBasedCost(HiveRelNode hr) { - return new HiveCost(hr.getRows(), 0, 0); + public static RelOptCost computeCardinalityBasedCost(HiveRelNode hr, RelMetadataQuery mq) { + return new HiveCost(mq.getRowCount(hr), 0, 0); } public HiveCost computeScanCost(double cardinality, double avgTupleSize) { http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveCostModel.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveCostModel.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveCostModel.java index 4af1f8d..3783d5f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveCostModel.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveCostModel.java @@ -23,13 +23,14 @@ import org.apache.calcite.plan.RelOptCost; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.rel.RelCollation; import org.apache.calcite.rel.RelDistribution; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan; import com.google.common.collect.ImmutableList; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Cost model interface. @@ -49,7 +50,7 @@ public abstract class HiveCostModel { public abstract RelOptCost getAggregateCost(HiveAggregate aggregate); - public abstract RelOptCost getScanCost(HiveTableScan ts); + public abstract RelOptCost getScanCost(HiveTableScan ts, RelMetadataQuery mq); public RelOptCost getJoinCost(HiveJoin join) { // Select algorithm with min cost http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveDefaultCostModel.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveDefaultCostModel.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveDefaultCostModel.java index 40f2cef..11b6aa3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveDefaultCostModel.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveDefaultCostModel.java @@ -55,7 +55,7 @@ public class HiveDefaultCostModel extends HiveCostModel { } @Override - public RelOptCost getScanCost(HiveTableScan ts) { + public RelOptCost getScanCost(HiveTableScan ts, RelMetadataQuery mq) { return HiveCost.FACTORY.makeZeroCost(); } http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveOnTezCostModel.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveOnTezCostModel.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveOnTezCostModel.java index af5fc5e..2dbfd8f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveOnTezCostModel.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveOnTezCostModel.java @@ -78,8 +78,8 @@ public class HiveOnTezCostModel extends HiveCostModel { } @Override - public RelOptCost getScanCost(HiveTableScan ts) { - return algoUtils.computeScanCost(ts.getRows(), RelMetadataQuery.instance().getAverageRowSize(ts)); + public RelOptCost getScanCost(HiveTableScan ts, RelMetadataQuery mq) { + return algoUtils.computeScanCost(mq.getRowCount(ts), mq.getAverageRowSize(ts)); } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveRelMdCost.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveRelMdCost.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveRelMdCost.java index cbea307..c6b8ce3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveRelMdCost.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveRelMdCost.java @@ -67,7 +67,7 @@ public class HiveRelMdCost implements MetadataHandler { private final RelNode childRel; private final double childCardinality; + private final RelMetadataQuery mq; - protected FilterSelectivityEstimator(RelNode childRel) { + protected FilterSelectivityEstimator(RelNode childRel, RelMetadataQuery mq) { super(true); + this.mq = mq; this.childRel = childRel; - this.childCardinality = RelMetadataQuery.instance().getRowCount(childRel); + this.childCardinality = mq.getRowCount(childRel); } public Double estimateSelectivity(RexNode predicate) { @@ -91,7 +93,7 @@ public class FilterSelectivityEstimator extends RexVisitorImpl { case IS_NOT_NULL: { if (childRel instanceof HiveTableScan) { double noOfNulls = getMaxNulls(call, (HiveTableScan) childRel); - double totalNoOfTuples = childRel.getRows(); + double totalNoOfTuples = mq.getRowCount(childRel); if (totalNoOfTuples >= noOfNulls) { selectivity = (totalNoOfTuples - noOfNulls) / Math.max(totalNoOfTuples, 1); } else { @@ -252,7 +254,6 @@ public class FilterSelectivityEstimator extends RexVisitorImpl { double tmpNDV; double maxNDV = 1.0; InputReferencedVisitor irv; - RelMetadataQuery mq = RelMetadataQuery.instance(); for (RexNode op : call.getOperands()) { if (op instanceof RexInputRef) { tmpNDV = HiveRelMdDistinctRowCount.getDistinctRowCount(this.childRel, mq, http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistinctRowCount.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistinctRowCount.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistinctRowCount.java index 77f7aa8..be2d5b3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistinctRowCount.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistinctRowCount.java @@ -81,7 +81,7 @@ public class HiveRelMdDistinctRowCount extends RelMdDistinctRowCount { noDistinctRows *= cStat.getCountDistint(); } - return Math.min(noDistinctRows, htRel.getRows()); + return Math.min(noDistinctRows, mq.getRowCount(htRel)); } public static Double getDistinctRowCount(RelNode r, RelMetadataQuery mq, int indx) { http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdRowCount.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdRowCount.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdRowCount.java index 7bba80b..9199d52 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdRowCount.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdRowCount.java @@ -72,7 +72,8 @@ public class HiveRelMdRowCount extends RelMdRowCount { } return pkfk.fkInfo.rowCount * selectivity; } - return join.getRows(); + // Do not call mq.getRowCount(join), will trigger CyclicMetadataException + return join.estimateRowCount(mq); } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java index 7192684..046f51b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java @@ -56,7 +56,7 @@ public class HiveRelMdSelectivity extends RelMdSelectivity { public Double getSelectivity(HiveTableScan t, RelMetadataQuery mq, RexNode predicate) { if (predicate != null) { - FilterSelectivityEstimator filterSelEstmator = new FilterSelectivityEstimator(t); + FilterSelectivityEstimator filterSelEstmator = new FilterSelectivityEstimator(t, mq); return filterSelEstmator.estimateSelectivity(predicate); } @@ -85,7 +85,7 @@ public class HiveRelMdSelectivity extends RelMdSelectivity { getCombinedPredicateForJoin(j, predicate); if (!predInfo.getKey()) { return - new FilterSelectivityEstimator(j). + new FilterSelectivityEstimator(j, mq). estimateSelectivity(predInfo.getValue()); } http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java index 9a5a2ba..aa4d8a5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java @@ -138,7 +138,7 @@ public class HiveRelMdUniqueKeys implements MetadataHandler colStats = tScan.getColStat(BitSets .toList(projectedCols)); Set keys = new HashSet();