Repository: hive Updated Branches: refs/heads/master bd453b8e1 -> f404b0ddf
HIVE-20598: Fix typos in HiveAlgorithmsUtil calculations (Zoltan Haindrich reviewed by Ashutosh Chauhan) Signed-off-by: Zoltan Haindrich <k...@rxd.hu> Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f404b0dd Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f404b0dd Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f404b0dd Branch: refs/heads/master Commit: f404b0ddf9b0432429177351818da7649edf48bd Parents: bd453b8 Author: Zoltan Haindrich <k...@rxd.hu> Authored: Fri Sep 21 17:28:21 2018 +0200 Committer: Zoltan Haindrich <k...@rxd.hu> Committed: Fri Sep 21 17:28:21 2018 +0200 ---------------------------------------------------------------------- .../calcite/cost/HiveAlgorithmsUtil.java | 30 ++++++++++---------- .../calcite/cost/HiveOnTezCostModel.java | 12 ++++---- 2 files changed, 21 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/f404b0dd/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveAlgorithmsUtil.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveAlgorithmsUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveAlgorithmsUtil.java index 8b87ae4..579a2e7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveAlgorithmsUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveAlgorithmsUtil.java @@ -39,7 +39,7 @@ import com.google.common.collect.ImmutableList; public class HiveAlgorithmsUtil { - private final double cpuCost; + private final double cpuUnitCost; private final double netCost; private final double localFSWrite; private final double localFSRead; @@ -47,8 +47,8 @@ public class HiveAlgorithmsUtil { private final double hdfsRead; HiveAlgorithmsUtil(HiveConf conf) { - cpuCost = Double.parseDouble(HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_CBO_COST_MODEL_CPU)); - netCost = cpuCost + cpuUnitCost = Double.parseDouble(HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_CBO_COST_MODEL_CPU)); + netCost = cpuUnitCost * Double.parseDouble(HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_CBO_COST_MODEL_NET)); localFSWrite = netCost * Double.parseDouble(HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_CBO_COST_MODEL_LFS_WRITE)); @@ -80,13 +80,13 @@ public class HiveAlgorithmsUtil { cpuCost += computeSortCPUCost(cardinality); } // Merge cost - cpuCost += cardinality * cpuCost; + cpuCost += cardinality * cpuUnitCost; } return cpuCost; } public double computeSortCPUCost(Double cardinality) { - return cardinality * Math.log(cardinality) * cpuCost; + return cardinality * Math.log(cardinality) * cpuUnitCost; } public double computeSortMergeIOCost( @@ -113,7 +113,7 @@ public class HiveAlgorithmsUtil { return ioCost; } - public static double computeMapJoinCPUCost( + public double computeMapJoinCPUCost( ImmutableList<Double> cardinalities, ImmutableBitSet streaming) { // Hash-join @@ -123,7 +123,7 @@ public class HiveAlgorithmsUtil { if (!streaming.get(i)) { cpuCost += cardinality; } - cpuCost += cardinality * cpuCost; + cpuCost += cardinality * cpuUnitCost; } return cpuCost; } @@ -151,9 +151,9 @@ public class HiveAlgorithmsUtil { for (int i=0; i<cardinalities.size(); i++) { double cardinality = cardinalities.get(i); if (!streaming.get(i)) { - cpuCost += cardinality * cpuCost; + cpuCost += cardinality * cpuUnitCost; } - cpuCost += cardinality * cpuCost; + cpuCost += cardinality * cpuUnitCost; } return cpuCost; } @@ -173,12 +173,12 @@ public class HiveAlgorithmsUtil { return ioCost; } - public static double computeSMBMapJoinCPUCost( + public double computeSMBMapJoinCPUCost( ImmutableList<Double> cardinalities) { // Hash-join double cpuCost = 0.0; for (int i=0; i<cardinalities.size(); i++) { - cpuCost += cardinalities.get(i) * cpuCost; + cpuCost += cardinalities.get(i) * cpuUnitCost; } return cpuCost; } @@ -225,12 +225,12 @@ public class HiveAlgorithmsUtil { for (int leftPos : joinLeafPredInfo.getProjsFromLeftPartOfJoinKeysInJoinSchema()) { final RelFieldCollation leftFieldCollation = new RelFieldCollation(leftPos); collationListBuilder.add(leftFieldCollation); - leftCollationListBuilder.add(leftFieldCollation); + leftCollationListBuilder.add(leftFieldCollation); } for (int rightPos : joinLeafPredInfo.getProjsFromRightPartOfJoinKeysInJoinSchema()) { final RelFieldCollation rightFieldCollation = new RelFieldCollation(rightPos); collationListBuilder.add(rightFieldCollation); - rightCollationListBuilder.add(rightFieldCollation); + rightCollationListBuilder.add(rightFieldCollation); } } @@ -285,10 +285,10 @@ public class HiveAlgorithmsUtil { JoinLeafPredicateInfo joinLeafPredInfo = joinPredInfo. getEquiJoinPredicateElements().get(i); for (int leftPos : joinLeafPredInfo.getProjsFromLeftPartOfJoinKeysInJoinSchema()) { - leftKeysListBuilder.add(leftPos); + leftKeysListBuilder.add(leftPos); } for (int rightPos : joinLeafPredInfo.getProjsFromRightPartOfJoinKeysInJoinSchema()) { - rightKeysListBuilder.add(rightPos); + rightKeysListBuilder.add(rightPos); } } http://git-wip-us.apache.org/repos/asf/hive/blob/f404b0dd/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveOnTezCostModel.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveOnTezCostModel.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveOnTezCostModel.java index 1fc9ea4..b451b13 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveOnTezCostModel.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveOnTezCostModel.java @@ -23,22 +23,22 @@ import java.util.List; import org.apache.calcite.plan.RelOptCost; import org.apache.calcite.rel.RelCollation; import org.apache.calcite.rel.RelDistribution; -import org.apache.calcite.rel.RelDistributions; import org.apache.calcite.rel.RelDistribution.Type; +import org.apache.calcite.rel.RelDistributions; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.ImmutableIntList; import org.apache.calcite.util.Pair; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil.JoinPredicateInfo; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin; -import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin.MapJoinStreamingRelation; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; @@ -267,7 +267,7 @@ public class HiveOnTezCostModel extends HiveCostModel { return null; } ImmutableBitSet streaming = streamingBuilder.build(); - final double cpuCost = HiveAlgorithmsUtil.computeMapJoinCPUCost(cardinalities, streaming); + final double cpuCost = algoUtils.computeMapJoinCPUCost(cardinalities, streaming); // 3. IO cost = cost of transferring small tables to join node * // degree of parallelism final Double leftRAverageSize = mq.getAverageRowSize(join.getLeft()); @@ -599,7 +599,7 @@ public class HiveOnTezCostModel extends HiveCostModel { return null; } ImmutableBitSet streaming = streamingBuilder.build(); - final double cpuCost = HiveAlgorithmsUtil.computeSMBMapJoinCPUCost(cardinalities); + final double cpuCost = algoUtils.computeSMBMapJoinCPUCost(cardinalities); // 3. IO cost = cost of transferring small tables to join node * // degree of parallelism final Double leftRAverageSize = mq.getAverageRowSize(join.getLeft());
