hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject [01/36] hive git commit: HIVE-16757 Remove calls to deprecated AbstractRelNode.getRows (Remus Rusanu reviewed by Ashutosh Chauhan)
Date Tue, 06 Jun 2017 18:35:28 GMT
Repository: hive
Updated Branches:
  refs/heads/hive-14535 d9e4ae09e -> 78f28c885


HIVE-16757 Remove calls to deprecated AbstractRelNode.getRows (Remus Rusanu reviewed by Ashutosh
Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8aee8d4f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8aee8d4f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8aee8d4f

Branch: refs/heads/hive-14535
Commit: 8aee8d4f2b124fcfa093724b4de0a54287a8084f
Parents: 86b1877
Author: Remus Rusanu <remusr@apache.org>
Authored: Tue May 30 22:37:54 2017 -0700
Committer: Remus Rusanu <remusr@apache.org>
Committed: Tue May 30 22:37:54 2017 -0700

----------------------------------------------------------------------
 .../hive/ql/optimizer/calcite/cost/HiveAlgorithmsUtil.java  | 4 ++--
 .../hive/ql/optimizer/calcite/cost/HiveCostModel.java       | 7 ++++---
 .../ql/optimizer/calcite/cost/HiveDefaultCostModel.java     | 2 +-
 .../hive/ql/optimizer/calcite/cost/HiveOnTezCostModel.java  | 4 ++--
 .../hive/ql/optimizer/calcite/cost/HiveRelMdCost.java       | 2 +-
 .../optimizer/calcite/stats/FilterSelectivityEstimator.java | 9 +++++----
 .../optimizer/calcite/stats/HiveRelMdDistinctRowCount.java  | 2 +-
 .../hive/ql/optimizer/calcite/stats/HiveRelMdRowCount.java  | 3 ++-
 .../ql/optimizer/calcite/stats/HiveRelMdSelectivity.java    | 4 ++--
 .../ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java     | 2 +-
 10 files changed, 21 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveAlgorithmsUtil.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveAlgorithmsUtil.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveAlgorithmsUtil.java
index 4ba10e2..28c7362 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveAlgorithmsUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveAlgorithmsUtil.java
@@ -60,8 +60,8 @@ public class HiveAlgorithmsUtil {
         * Double.parseDouble(HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_CBO_COST_MODEL_HDFS_READ));
   }
 
-  public static RelOptCost computeCardinalityBasedCost(HiveRelNode hr) {
-    return new HiveCost(hr.getRows(), 0, 0);
+  public static RelOptCost computeCardinalityBasedCost(HiveRelNode hr, RelMetadataQuery mq)
{
+    return new HiveCost(mq.getRowCount(hr), 0, 0);
   }
 
   public HiveCost computeScanCost(double cardinality, double avgTupleSize) {

http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveCostModel.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveCostModel.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveCostModel.java
index 4af1f8d..3783d5f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveCostModel.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveCostModel.java
@@ -23,13 +23,14 @@ import org.apache.calcite.plan.RelOptCost;
 import org.apache.calcite.plan.RelOptUtil;
 import org.apache.calcite.rel.RelCollation;
 import org.apache.calcite.rel.RelDistribution;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
 import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate;
 import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin;
 import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan;
 
 import com.google.common.collect.ImmutableList;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Cost model interface.
@@ -49,7 +50,7 @@ public abstract class HiveCostModel {
 
   public abstract RelOptCost getAggregateCost(HiveAggregate aggregate);
 
-  public abstract RelOptCost getScanCost(HiveTableScan ts);
+  public abstract RelOptCost getScanCost(HiveTableScan ts, RelMetadataQuery mq);
 
   public RelOptCost getJoinCost(HiveJoin join) {
     // Select algorithm with min cost

http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveDefaultCostModel.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveDefaultCostModel.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveDefaultCostModel.java
index 40f2cef..11b6aa3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveDefaultCostModel.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveDefaultCostModel.java
@@ -55,7 +55,7 @@ public class HiveDefaultCostModel extends HiveCostModel {
   }
 
   @Override
-  public RelOptCost getScanCost(HiveTableScan ts) {
+  public RelOptCost getScanCost(HiveTableScan ts, RelMetadataQuery mq) {
     return HiveCost.FACTORY.makeZeroCost();
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveOnTezCostModel.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveOnTezCostModel.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveOnTezCostModel.java
index af5fc5e..2dbfd8f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveOnTezCostModel.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveOnTezCostModel.java
@@ -78,8 +78,8 @@ public class HiveOnTezCostModel extends HiveCostModel {
   }
 
   @Override
-  public RelOptCost getScanCost(HiveTableScan ts) {
-    return algoUtils.computeScanCost(ts.getRows(), RelMetadataQuery.instance().getAverageRowSize(ts));
+  public RelOptCost getScanCost(HiveTableScan ts, RelMetadataQuery mq) {
+    return algoUtils.computeScanCost(mq.getRowCount(ts), mq.getAverageRowSize(ts));
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveRelMdCost.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveRelMdCost.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveRelMdCost.java
index cbea307..c6b8ce3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveRelMdCost.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveRelMdCost.java
@@ -67,7 +67,7 @@ public class HiveRelMdCost implements MetadataHandler<BuiltInMetadata.NonCumulat
   }
 
   public RelOptCost getNonCumulativeCost(HiveTableScan ts, RelMetadataQuery mq) {
-    return hiveCostModel.getScanCost(ts);
+    return hiveCostModel.getScanCost(ts, mq);
   }
 
   // Default case

http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java
index a25b58b..9b8e6a4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java
@@ -44,11 +44,13 @@ import org.apache.hadoop.hive.ql.plan.ColStatistics;
 public class FilterSelectivityEstimator extends RexVisitorImpl<Double> {
   private final RelNode childRel;
   private final double  childCardinality;
+  private final RelMetadataQuery mq;
 
-  protected FilterSelectivityEstimator(RelNode childRel) {
+  protected FilterSelectivityEstimator(RelNode childRel, RelMetadataQuery mq) {
     super(true);
+    this.mq = mq;
     this.childRel = childRel;
-    this.childCardinality = RelMetadataQuery.instance().getRowCount(childRel);
+    this.childCardinality = mq.getRowCount(childRel);
   }
 
   public Double estimateSelectivity(RexNode predicate) {
@@ -91,7 +93,7 @@ public class FilterSelectivityEstimator extends RexVisitorImpl<Double>
{
     case IS_NOT_NULL: {
       if (childRel instanceof HiveTableScan) {
         double noOfNulls = getMaxNulls(call, (HiveTableScan) childRel);
-        double totalNoOfTuples = childRel.getRows();
+        double totalNoOfTuples = mq.getRowCount(childRel);
         if (totalNoOfTuples >= noOfNulls) {
           selectivity = (totalNoOfTuples - noOfNulls) / Math.max(totalNoOfTuples, 1);
         } else {
@@ -252,7 +254,6 @@ public class FilterSelectivityEstimator extends RexVisitorImpl<Double>
{
     double tmpNDV;
     double maxNDV = 1.0;
     InputReferencedVisitor irv;
-    RelMetadataQuery mq = RelMetadataQuery.instance();
     for (RexNode op : call.getOperands()) {
       if (op instanceof RexInputRef) {
         tmpNDV = HiveRelMdDistinctRowCount.getDistinctRowCount(this.childRel, mq,

http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistinctRowCount.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistinctRowCount.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistinctRowCount.java
index 77f7aa8..be2d5b3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistinctRowCount.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistinctRowCount.java
@@ -81,7 +81,7 @@ public class HiveRelMdDistinctRowCount extends RelMdDistinctRowCount {
       noDistinctRows *= cStat.getCountDistint();
     }
 
-    return Math.min(noDistinctRows, htRel.getRows());
+    return Math.min(noDistinctRows, mq.getRowCount(htRel));
   }
 
   public static Double getDistinctRowCount(RelNode r, RelMetadataQuery mq, int indx) {

http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdRowCount.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdRowCount.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdRowCount.java
index 7bba80b..9199d52 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdRowCount.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdRowCount.java
@@ -72,7 +72,8 @@ public class HiveRelMdRowCount extends RelMdRowCount {
       }
       return pkfk.fkInfo.rowCount * selectivity;
     }
-    return join.getRows();
+    // Do not call mq.getRowCount(join), will trigger CyclicMetadataException
+    return join.estimateRowCount(mq);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java
index 7192684..046f51b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java
@@ -56,7 +56,7 @@ public class HiveRelMdSelectivity extends RelMdSelectivity {
 
   public Double getSelectivity(HiveTableScan t, RelMetadataQuery mq, RexNode predicate) {
     if (predicate != null) {
-      FilterSelectivityEstimator filterSelEstmator = new FilterSelectivityEstimator(t);
+      FilterSelectivityEstimator filterSelEstmator = new FilterSelectivityEstimator(t, mq);
       return filterSelEstmator.estimateSelectivity(predicate);
     }
 
@@ -85,7 +85,7 @@ public class HiveRelMdSelectivity extends RelMdSelectivity {
         getCombinedPredicateForJoin(j, predicate);
     if (!predInfo.getKey()) {
       return
-          new FilterSelectivityEstimator(j).
+          new FilterSelectivityEstimator(j, mq).
           estimateSelectivity(predInfo.getValue());
     }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java
index 9a5a2ba..aa4d8a5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java
@@ -138,7 +138,7 @@ public class HiveRelMdUniqueKeys implements MetadataHandler<BuiltInMetadata.Uniq
       projectPos++;
     }
 
-    double numRows = tScan.getRows();
+    double numRows = mq.getRowCount(tScan);
     List<ColStatistics> colStats = tScan.getColStat(BitSets
         .toList(projectedCols));
     Set<ImmutableBitSet> keys = new HashSet<ImmutableBitSet>();


Mime
View raw message