Repository: hive Updated Branches: refs/heads/hive-14535 d9e4ae09e -> 78f28c885
HIVE-16757 Remove calls to deprecated AbstractRelNode.getRows (Remus Rusanu reviewed by Ashutosh Chauhan) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8aee8d4f Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8aee8d4f Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8aee8d4f Branch: refs/heads/hive-14535 Commit: 8aee8d4f2b124fcfa093724b4de0a54287a8084f Parents: 86b1877 Author: Remus Rusanu <rem...@apache.org> Authored: Tue May 30 22:37:54 2017 -0700 Committer: Remus Rusanu <rem...@apache.org> Committed: Tue May 30 22:37:54 2017 -0700 ---------------------------------------------------------------------- .../hive/ql/optimizer/calcite/cost/HiveAlgorithmsUtil.java | 4 ++-- .../hive/ql/optimizer/calcite/cost/HiveCostModel.java | 7 ++++--- .../ql/optimizer/calcite/cost/HiveDefaultCostModel.java | 2 +- .../hive/ql/optimizer/calcite/cost/HiveOnTezCostModel.java | 4 ++-- .../hive/ql/optimizer/calcite/cost/HiveRelMdCost.java | 2 +- .../optimizer/calcite/stats/FilterSelectivityEstimator.java | 9 +++++---- .../optimizer/calcite/stats/HiveRelMdDistinctRowCount.java | 2 +- .../hive/ql/optimizer/calcite/stats/HiveRelMdRowCount.java | 3 ++- .../ql/optimizer/calcite/stats/HiveRelMdSelectivity.java | 4 ++-- .../ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java | 2 +- 10 files changed, 21 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveAlgorithmsUtil.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveAlgorithmsUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveAlgorithmsUtil.java index 4ba10e2..28c7362 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveAlgorithmsUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveAlgorithmsUtil.java @@ -60,8 +60,8 @@ public class HiveAlgorithmsUtil { * Double.parseDouble(HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_CBO_COST_MODEL_HDFS_READ)); } - public static RelOptCost computeCardinalityBasedCost(HiveRelNode hr) { - return new HiveCost(hr.getRows(), 0, 0); + public static RelOptCost computeCardinalityBasedCost(HiveRelNode hr, RelMetadataQuery mq) { + return new HiveCost(mq.getRowCount(hr), 0, 0); } public HiveCost computeScanCost(double cardinality, double avgTupleSize) { http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveCostModel.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveCostModel.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveCostModel.java index 4af1f8d..3783d5f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveCostModel.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveCostModel.java @@ -23,13 +23,14 @@ import org.apache.calcite.plan.RelOptCost; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.rel.RelCollation; import org.apache.calcite.rel.RelDistribution; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan; import com.google.common.collect.ImmutableList; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Cost model interface. @@ -49,7 +50,7 @@ public abstract class HiveCostModel { public abstract RelOptCost getAggregateCost(HiveAggregate aggregate); - public abstract RelOptCost getScanCost(HiveTableScan ts); + public abstract RelOptCost getScanCost(HiveTableScan ts, RelMetadataQuery mq); public RelOptCost getJoinCost(HiveJoin join) { // Select algorithm with min cost http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveDefaultCostModel.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveDefaultCostModel.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveDefaultCostModel.java index 40f2cef..11b6aa3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveDefaultCostModel.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveDefaultCostModel.java @@ -55,7 +55,7 @@ public class HiveDefaultCostModel extends HiveCostModel { } @Override - public RelOptCost getScanCost(HiveTableScan ts) { + public RelOptCost getScanCost(HiveTableScan ts, RelMetadataQuery mq) { return HiveCost.FACTORY.makeZeroCost(); } http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveOnTezCostModel.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveOnTezCostModel.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveOnTezCostModel.java index af5fc5e..2dbfd8f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveOnTezCostModel.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveOnTezCostModel.java @@ -78,8 +78,8 @@ public class HiveOnTezCostModel extends HiveCostModel { } @Override - public RelOptCost getScanCost(HiveTableScan ts) { - return algoUtils.computeScanCost(ts.getRows(), RelMetadataQuery.instance().getAverageRowSize(ts)); + public RelOptCost getScanCost(HiveTableScan ts, RelMetadataQuery mq) { + return algoUtils.computeScanCost(mq.getRowCount(ts), mq.getAverageRowSize(ts)); } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveRelMdCost.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveRelMdCost.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveRelMdCost.java index cbea307..c6b8ce3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveRelMdCost.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveRelMdCost.java @@ -67,7 +67,7 @@ public class HiveRelMdCost implements MetadataHandler<BuiltInMetadata.NonCumulat } public RelOptCost getNonCumulativeCost(HiveTableScan ts, RelMetadataQuery mq) { - return hiveCostModel.getScanCost(ts); + return hiveCostModel.getScanCost(ts, mq); } // Default case http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java index a25b58b..9b8e6a4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java @@ -44,11 +44,13 @@ import org.apache.hadoop.hive.ql.plan.ColStatistics; public class FilterSelectivityEstimator extends RexVisitorImpl<Double> { private final RelNode childRel; private final double childCardinality; + private final RelMetadataQuery mq; - protected FilterSelectivityEstimator(RelNode childRel) { + protected FilterSelectivityEstimator(RelNode childRel, RelMetadataQuery mq) { super(true); + this.mq = mq; this.childRel = childRel; - this.childCardinality = RelMetadataQuery.instance().getRowCount(childRel); + this.childCardinality = mq.getRowCount(childRel); } public Double estimateSelectivity(RexNode predicate) { @@ -91,7 +93,7 @@ public class FilterSelectivityEstimator extends RexVisitorImpl<Double> { case IS_NOT_NULL: { if (childRel instanceof HiveTableScan) { double noOfNulls = getMaxNulls(call, (HiveTableScan) childRel); - double totalNoOfTuples = childRel.getRows(); + double totalNoOfTuples = mq.getRowCount(childRel); if (totalNoOfTuples >= noOfNulls) { selectivity = (totalNoOfTuples - noOfNulls) / Math.max(totalNoOfTuples, 1); } else { @@ -252,7 +254,6 @@ public class FilterSelectivityEstimator extends RexVisitorImpl<Double> { double tmpNDV; double maxNDV = 1.0; InputReferencedVisitor irv; - RelMetadataQuery mq = RelMetadataQuery.instance(); for (RexNode op : call.getOperands()) { if (op instanceof RexInputRef) { tmpNDV = HiveRelMdDistinctRowCount.getDistinctRowCount(this.childRel, mq, http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistinctRowCount.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistinctRowCount.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistinctRowCount.java index 77f7aa8..be2d5b3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistinctRowCount.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistinctRowCount.java @@ -81,7 +81,7 @@ public class HiveRelMdDistinctRowCount extends RelMdDistinctRowCount { noDistinctRows *= cStat.getCountDistint(); } - return Math.min(noDistinctRows, htRel.getRows()); + return Math.min(noDistinctRows, mq.getRowCount(htRel)); } public static Double getDistinctRowCount(RelNode r, RelMetadataQuery mq, int indx) { http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdRowCount.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdRowCount.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdRowCount.java index 7bba80b..9199d52 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdRowCount.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdRowCount.java @@ -72,7 +72,8 @@ public class HiveRelMdRowCount extends RelMdRowCount { } return pkfk.fkInfo.rowCount * selectivity; } - return join.getRows(); + // Do not call mq.getRowCount(join), will trigger CyclicMetadataException + return join.estimateRowCount(mq); } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java index 7192684..046f51b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java @@ -56,7 +56,7 @@ public class HiveRelMdSelectivity extends RelMdSelectivity { public Double getSelectivity(HiveTableScan t, RelMetadataQuery mq, RexNode predicate) { if (predicate != null) { - FilterSelectivityEstimator filterSelEstmator = new FilterSelectivityEstimator(t); + FilterSelectivityEstimator filterSelEstmator = new FilterSelectivityEstimator(t, mq); return filterSelEstmator.estimateSelectivity(predicate); } @@ -85,7 +85,7 @@ public class HiveRelMdSelectivity extends RelMdSelectivity { getCombinedPredicateForJoin(j, predicate); if (!predInfo.getKey()) { return - new FilterSelectivityEstimator(j). + new FilterSelectivityEstimator(j, mq). estimateSelectivity(predInfo.getValue()); } http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java index 9a5a2ba..aa4d8a5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java @@ -138,7 +138,7 @@ public class HiveRelMdUniqueKeys implements MetadataHandler<BuiltInMetadata.Uniq projectPos++; } - double numRows = tScan.getRows(); + double numRows = mq.getRowCount(tScan); List<ColStatistics> colStats = tScan.getColStat(BitSets .toList(projectedCols)); Set<ImmutableBitSet> keys = new HashSet<ImmutableBitSet>();
