HIVE-17817 : Stabilize crossproduct warning message output order (Zoltan Haindrich via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <hashut...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/077a9e5b Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/077a9e5b Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/077a9e5b Branch: refs/heads/master Commit: 077a9e5b4d6566744498468be0e028f93514eb61 Parents: 39e1636 Author: Zoltan Haindrich <k...@rxd.hu> Authored: Thu Oct 19 17:19:09 2017 -0700 Committer: Ashutosh Chauhan <hashut...@apache.org> Committed: Thu Oct 19 18:11:14 2017 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hive/cli/TestTezPerfCliDriver.java | 14 +++++++++++++- .../hadoop/hive/cli/control/CorePerfCliDriver.java | 8 ++++++++ .../hive/ql/optimizer/physical/CrossProductCheck.java | 4 ++-- .../ql/optimizer/physical/SparkCrossProductCheck.java | 5 ++--- .../java/org/apache/hadoop/hive/ql/plan/TezWork.java | 6 +++--- ql/src/test/queries/clientpositive/perf/query28.q | 2 ++ .../results/clientpositive/perf/tez/query14.q.out | 2 +- 7 files changed, 31 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/077a9e5b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestTezPerfCliDriver.java ---------------------------------------------------------------------- diff --git a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestTezPerfCliDriver.java b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestTezPerfCliDriver.java index 0c9b2ba..69578fc 100644 --- a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestTezPerfCliDriver.java +++ b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestTezPerfCliDriver.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.cli; import java.io.File; +import java.util.Comparator; import java.util.List; import org.apache.hadoop.hive.cli.control.CliAdapter; @@ -37,7 +38,18 @@ public class TestTezPerfCliDriver { @Parameters(name = "{0}") public static List<Object[]> getParameters() throws Exception { - return adapter.getParameters(); + List<Object[]> parameters = adapter.getParameters(); + parameters.sort(new C1()); + return parameters; + } + + static class C1 implements Comparator<Object[]> { + + @Override + public int compare(Object[] o1, Object[] o2) { + return o1[0].toString().compareTo(o2[0].toString()); + } + } @ClassRule http://git-wip-us.apache.org/repos/asf/hive/blob/077a9e5b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java index d80bd44..7cfead8 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java @@ -86,6 +86,14 @@ public class CorePerfCliDriver extends CliAdapter{ @Override public void setUp() { + try { + qt.clearPostTestEffects(); + } catch (Exception e) { + System.err.println("Exception: " + e.getMessage()); + e.printStackTrace(); + System.err.flush(); + fail("Unexpected exception"); + } } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/077a9e5b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CrossProductCheck.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CrossProductCheck.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CrossProductCheck.java index f5abaf1..4b35bb6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CrossProductCheck.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CrossProductCheck.java @@ -27,6 +27,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Stack; +import java.util.TreeMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -180,8 +181,7 @@ public class CrossProductCheck implements PhysicalPlanResolver, Dispatcher { ReduceWork rWork = (ReduceWork) wrk; Operator<? extends OperatorDesc> reducer = ((ReduceWork)wrk).getReducer(); if ( reducer instanceof JoinOperator || reducer instanceof CommonMergeJoinOperator ) { - Map<Integer, ExtractReduceSinkInfo.Info> rsInfo = - new HashMap<Integer, ExtractReduceSinkInfo.Info>(); + Map<Integer, ExtractReduceSinkInfo.Info> rsInfo = new TreeMap<Integer, ExtractReduceSinkInfo.Info>(); for(Map.Entry<Integer, String> e : rWork.getTagToInput().entrySet()) { rsInfo.putAll(getReducerInfo(tzWrk, rWork.getName(), e.getValue())); } http://git-wip-us.apache.org/repos/asf/hive/blob/077a9e5b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SparkCrossProductCheck.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SparkCrossProductCheck.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SparkCrossProductCheck.java index f904451..7f3b1b3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SparkCrossProductCheck.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SparkCrossProductCheck.java @@ -36,11 +36,11 @@ import org.apache.hadoop.hive.ql.session.SessionState; import java.io.Serializable; import java.util.ArrayList; -import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Stack; +import java.util.TreeMap; /** * Check each MapJoin and ShuffleJoin Operator to see if they are performing a cross product. @@ -92,8 +92,7 @@ public class SparkCrossProductCheck implements PhysicalPlanResolver, Dispatcher for (ReduceWork reduceWork : sparkWork.getAllReduceWork()) { Operator<? extends OperatorDesc> reducer = reduceWork.getReducer(); if (reducer instanceof JoinOperator || reducer instanceof CommonMergeJoinOperator) { - Map<Integer, CrossProductCheck.ExtractReduceSinkInfo.Info> rsInfo = - new HashMap<Integer, CrossProductCheck.ExtractReduceSinkInfo.Info>(); + Map<Integer, CrossProductCheck.ExtractReduceSinkInfo.Info> rsInfo = new TreeMap<Integer, CrossProductCheck.ExtractReduceSinkInfo.Info>(); for (BaseWork parent : sparkWork.getParents(reduceWork)) { rsInfo.putAll(new CrossProductCheck.ExtractReduceSinkInfo(null).analyze(parent)); } http://git-wip-us.apache.org/repos/asf/hive/blob/077a9e5b/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java index a037ea3..18ff13b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java @@ -19,12 +19,12 @@ package org.apache.hadoop.hive.ql.plan; import java.io.Serializable; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -74,8 +74,8 @@ public class TezWork extends AbstractOperatorDesc { private static final AtomicInteger counter = new AtomicInteger(1); private final String dagId; private final String queryName; - private final Set<BaseWork> roots = new HashSet<BaseWork>(); - private final Set<BaseWork> leaves = new HashSet<BaseWork>(); + private final Set<BaseWork> roots = new LinkedHashSet<BaseWork>(); + private final Set<BaseWork> leaves = new LinkedHashSet<BaseWork>(); private final Map<BaseWork, List<BaseWork>> workGraph = new HashMap<BaseWork, List<BaseWork>>(); private final Map<BaseWork, List<BaseWork>> invertedWorkGraph = new HashMap<BaseWork, List<BaseWork>>(); private final Map<Pair<BaseWork, BaseWork>, TezEdgeProperty> edgeProperties = http://git-wip-us.apache.org/repos/asf/hive/blob/077a9e5b/ql/src/test/queries/clientpositive/perf/query28.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/perf/query28.q b/ql/src/test/queries/clientpositive/perf/query28.q index 169653e..fc3c1b2d 100644 --- a/ql/src/test/queries/clientpositive/perf/query28.q +++ b/ql/src/test/queries/clientpositive/perf/query28.q @@ -55,3 +55,5 @@ from (select avg(ss_list_price) B1_LP limit 100; -- end query 1 in stream 0 using template query28.tpl + +set hive.optimize.metadataonly=false; http://git-wip-us.apache.org/repos/asf/hive/blob/077a9e5b/ql/src/test/results/clientpositive/perf/tez/query14.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/perf/tez/query14.q.out b/ql/src/test/results/clientpositive/perf/tez/query14.q.out index 77959cb..b2a45f1 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query14.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query14.q.out @@ -1,6 +1,6 @@ Warning: Shuffle Join MERGEJOIN[890][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 5' is a cross product -Warning: Shuffle Join MERGEJOIN[892][tables = [$hdt$_2, $hdt$_3, $hdt$_1]] in Stage 'Reducer 16' is a cross product Warning: Shuffle Join MERGEJOIN[891][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 12' is a cross product +Warning: Shuffle Join MERGEJOIN[892][tables = [$hdt$_2, $hdt$_3, $hdt$_1]] in Stage 'Reducer 16' is a cross product PREHOOK: query: explain with cross_items as (select i_item_sk ss_item_sk
