HIVE-16296 : use LLAP executor count to configure reducer auto-parallelism (Sergey Shelukhin, Gopal Vijayaraghavan, reviewed by Gopal Vijayaraghavan, Sergey Shelukhin, Gunther Hagleitner)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/10bd23eb Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/10bd23eb Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/10bd23eb Branch: refs/heads/master Commit: 10bd23eb5c85948d7e246047818a628b9ab0a680 Parents: 656a99e Author: sergey <ser...@apache.org> Authored: Thu Apr 20 12:11:22 2017 -0700 Committer: sergey <ser...@apache.org> Committed: Thu Apr 20 12:31:01 2017 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/conf/HiveConf.java | 3 + .../llap/registry/impl/LlapRegistryService.java | 5 +- .../hadoop/hive/ql/exec/FetchOperator.java | 16 + .../ql/optimizer/SetReducerParallelism.java | 2 + .../physical/LlapClusterStateForCompile.java | 123 + .../hive/ql/optimizer/physical/LlapDecider.java | 56 +- .../hadoop/hive/ql/parse/GenTezUtils.java | 4 + .../hadoop/hive/ql/parse/TezCompiler.java | 5 +- .../apache/hadoop/hive/ql/plan/ReduceWork.java | 21 + .../hadoop/hive/ql/plan/TezEdgeProperty.java | 6 + .../queries/clientpositive/subquery_scalar.q | 1 + .../queries/clientpositive/unionDistinct_1.q | 4 +- .../clientpositive/vector_binary_join_groupby.q | 16 +- .../vector_groupby_grouping_sets_limit.q | 25 +- .../clientpositive/vector_groupby_reduce.q | 2 + .../clientpositive/llap/bucket_groupby.q.out | 422 +- .../clientpositive/llap/cbo_rp_lineage2.q.out | 58 +- .../clientpositive/llap/cbo_rp_views.q.out | 2 +- .../results/clientpositive/llap/cbo_views.q.out | 2 +- .../results/clientpositive/llap/cluster.q.out | 688 +- .../llap/correlationoptimizer1.q.out | 32 +- .../results/clientpositive/llap/cte_1.q.out | 37900 ++++++++--------- .../clientpositive/llap/except_distinct.q.out | 2 +- .../clientpositive/llap/explainuser_4.q.out | 4 +- .../llap/hybridgrace_hashjoin_2.q.out | 4 +- .../clientpositive/llap/intersect_all.q.out | 1470 +- .../llap/intersect_distinct.q.out | 924 +- .../clientpositive/llap/lateral_view.q.out | 8 +- .../results/clientpositive/llap/lineage2.q.out | 58 +- .../clientpositive/llap/multiMapJoin1.q.out | 4 +- .../clientpositive/llap/multi_column_in.q.out | 2 +- .../llap/multi_count_distinct_null.q.out | 24 +- .../clientpositive/llap/parquet_types.q.out | 2 +- .../llap/partition_multilevels.q.out | 244 +- .../llap/reduce_deduplicate_extended.q.out | 32 +- .../clientpositive/llap/skiphf_aggr.q.out | 4 +- .../clientpositive/llap/subquery_multi.q.out | 450 +- .../clientpositive/llap/subquery_notin.q.out | 106 +- .../clientpositive/llap/subquery_scalar.q.out | 408 +- .../llap/tez_dynpart_hashjoin_1.q.out | 2 +- .../clientpositive/llap/tez_union2.q.out | 924 +- .../llap/tez_vector_dynpart_hashjoin_1.q.out | 4 +- .../clientpositive/llap/unionDistinct_1.q.out | 4160 +- .../llap/vector_binary_join_groupby.q.out | 95 +- .../clientpositive/llap/vector_coalesce_2.q.out | 4 +- .../llap/vector_complex_all.q.out | 6 +- .../llap/vector_decimal_udf.q.out | 32 +- .../vector_groupby_grouping_sets_limit.q.out | 222 +- .../llap/vector_groupby_reduce.q.out | 33 + .../llap/vector_grouping_sets.q.out | 8 +- .../llap/vector_partitioned_date_time.q.out | 52 +- .../llap/vectorized_distinct_gby.q.out | 2 +- .../vectorized_dynamic_semijoin_reduction.q.out | 72 +- .../llap/vectorized_parquet.q.out | 164 +- .../vector_binary_join_groupby.q.out | 115 +- .../clientpositive/vector_groupby_reduce.q.out | 49 +- 56 files changed, 24817 insertions(+), 24266 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 457f7af..771f588 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2858,6 +2858,9 @@ public class HiveConf extends Configuration { "Turn on Tez' auto reducer parallelism feature. When enabled, Hive will still estimate data sizes\n" + "and set parallelism estimates. Tez will sample source vertices' output sizes and adjust the estimates at runtime as\n" + "necessary."), + TEZ_LLAP_MIN_REDUCER_PER_EXECUTOR("hive.tez.llap.min.reducer.per.executor", 0.95f, + "If above 0, the min number of reducers for auto-parallelism for LLAP scheduling will\n" + + "be set to this fraction of the number of executors."), TEZ_MAX_PARTITION_FACTOR("hive.tez.max.partition.factor", 2f, "When auto reducer parallelism is enabled this factor will be used to over-partition data in shuffle edges."), TEZ_MIN_PARTITION_FACTOR("hive.tez.min.partition.factor", 0.25f, http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapRegistryService.java ---------------------------------------------------------------------- diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapRegistryService.java b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapRegistryService.java index 610c0a5..76fc9c7 100644 --- a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapRegistryService.java +++ b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapRegistryService.java @@ -61,7 +61,7 @@ public class LlapRegistryService extends AbstractService { if (hosts.startsWith("@")) { // Caching instances only in case of the YARN registry. Each host based list will get it's own copy. String appName = hosts.substring(1); - String userName = HiveConf.getVar(conf, ConfVars.LLAP_ZK_REGISTRY_USER, RegistryUtils.currentUser()); + String userName = HiveConf.getVar(conf, ConfVars.LLAP_ZK_REGISTRY_USER, currentUser()); String key = appName + "-" + userName; registry = yarnRegistries.get(key); if (registry == null || !registry.isInState(STATE.STARTED)) { @@ -79,6 +79,9 @@ public class LlapRegistryService extends AbstractService { return registry; } + public static String currentUser() { + return RegistryUtils.currentUser(); + } @Override public void serviceInit(Configuration conf) { http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java index 004bb2f..13750cd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; +import java.util.Comparator; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -378,6 +379,9 @@ public class FetchOperator implements Serializable { inputSplits = splitSampling(work.getSplitSample(), inputSplits); } if (inputSplits.length > 0) { + if (HiveConf.getBoolVar(job, HiveConf.ConfVars.HIVE_IN_TEST)) { + Arrays.sort(inputSplits, new FetchInputFormatSplitComparator()); + } return inputSplits; } } @@ -695,4 +699,16 @@ public class FetchOperator implements Serializable { return inputFormat.getRecordReader(getInputSplit(), job, Reporter.NULL); } } + + private static class FetchInputFormatSplitComparator implements Comparator<FetchInputFormatSplit> { + @Override + public int compare(FetchInputFormatSplit a, FetchInputFormatSplit b) { + final Path ap = a.getPath(); + final Path bp = b.getPath(); + if (ap != null) { + return (ap.compareTo(bp)); + } + return Long.signum(a.getLength() - b.getLength()); + } + } } http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SetReducerParallelism.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SetReducerParallelism.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SetReducerParallelism.java index 60a8604..b51af55 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SetReducerParallelism.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SetReducerParallelism.java @@ -41,6 +41,7 @@ import org.apache.hadoop.hive.ql.stats.StatsUtils; import static org.apache.hadoop.hive.ql.plan.ReduceSinkDesc.ReducerTraits.AUTOPARALLEL; import static org.apache.hadoop.hive.ql.plan.ReduceSinkDesc.ReducerTraits.UNIFORM; +import static org.apache.hadoop.hive.ql.plan.ReduceSinkDesc.ReducerTraits.FIXED; /** * SetReducerParallelism determines how many reducers should @@ -106,6 +107,7 @@ public class SetReducerParallelism implements NodeProcessor { } } else { LOG.info("Number of reducers determined to be: "+desc.getNumReducers()); + desc.setReducerTraits(EnumSet.of(FIXED)); // usually controlled by bucketing } return false; http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapClusterStateForCompile.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapClusterStateForCompile.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapClusterStateForCompile.java new file mode 100644 index 0000000..b2e8614 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapClusterStateForCompile.java @@ -0,0 +1,123 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.optimizer.physical; + +import java.util.concurrent.ExecutionException; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.Callable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.llap.registry.ServiceInstance; +import org.apache.hadoop.hive.llap.registry.ServiceInstanceSet; +import org.apache.hadoop.hive.llap.registry.impl.InactiveServiceInstance; +import org.apache.hadoop.hive.llap.registry.impl.LlapRegistryService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class LlapClusterStateForCompile { + protected static final Logger LOG = LoggerFactory.getLogger(LlapClusterStateForCompile.class); + + private static final long CLUSTER_UPDATE_INTERVAL_NS = 120 * 1000000000L; // 2 minutes. + private Long lastClusterUpdateNs; + private Integer noConfigNodeCount, executorCount; + private LlapRegistryService svc; + private final Configuration conf; + + // It's difficult to impossible to pass global things to compilation, so we have a static cache. + private static final Cache<String, LlapClusterStateForCompile> CACHE = + CacheBuilder.newBuilder().initialCapacity(10).maximumSize(100).build(); + + public static LlapClusterStateForCompile getClusterInfo(final Configuration conf) { + final String nodes = HiveConf.getTrimmedVar(conf, HiveConf.ConfVars.LLAP_DAEMON_SERVICE_HOSTS); + final String userName = HiveConf.getVar( + conf, ConfVars.LLAP_ZK_REGISTRY_USER, LlapRegistryService.currentUser()); + Callable<LlapClusterStateForCompile> generator = new Callable<LlapClusterStateForCompile>() { + @Override + public LlapClusterStateForCompile call() throws Exception { + LOG.info("Creating cluster info for " + userName + ":" + nodes); + return new LlapClusterStateForCompile(conf); + } + }; + try { + return CACHE.get(userName + ":" + nodes, generator); + } catch (ExecutionException e) { + throw new RuntimeException(e); // Should never happen... ctor is just assignments. + } + } + + private LlapClusterStateForCompile(Configuration conf) { + this.conf = conf; + } + + public boolean hasClusterInfo() { + return lastClusterUpdateNs != null; + } + + public int getKnownExecutorCount() { + return executorCount; + } + + public int getNodeCountWithUnknownExecutors() { + return noConfigNodeCount; + } + + public synchronized void initClusterInfo() { + if (lastClusterUpdateNs != null) { + long elapsed = System.nanoTime() - lastClusterUpdateNs; + if (elapsed < CLUSTER_UPDATE_INTERVAL_NS) return; + } + if (svc == null) { + try { + svc = LlapRegistryService.getClient(conf); + } catch (Throwable t) { + LOG.info("Cannot create the client; ignoring", t); + return; // Don't fail; this is best-effort. + } + } + ServiceInstanceSet instances; + try { + instances = svc.getInstances(10); + } catch (IOException e) { + LOG.info("Cannot update cluster information; ignoring", e); + return; // Don't wait for the cluster if not started; this is best-effort. + } + int executorsLocal = 0, noConfigNodesLocal = 0; + for (ServiceInstance si : instances.getAll()) { + if (si instanceof InactiveServiceInstance) continue; // Shouldn't happen in getAll. + Map<String, String> props = si.getProperties(); + if (props == null) { + ++noConfigNodesLocal; + continue; + } + try { + executorsLocal += Integer.parseInt(props.get(ConfVars.LLAP_DAEMON_NUM_EXECUTORS.varname)); + } catch (NumberFormatException e) { + ++noConfigNodesLocal; + } + } + lastClusterUpdateNs = System.nanoTime(); + noConfigNodeCount = noConfigNodesLocal; + executorCount = executorsLocal; + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java index a694cf8..c81131e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java @@ -26,7 +26,6 @@ import static org.apache.hadoop.hive.ql.optimizer.physical.LlapDecider.LlapMode. import java.io.Serializable; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Deque; import java.util.EnumSet; @@ -51,7 +50,6 @@ import org.apache.hadoop.hive.ql.exec.SelectOperator; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.exec.tez.TezTask; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedInputFormatInterface; import org.apache.hadoop.hive.ql.io.HiveInputFormat; import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker; import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher; @@ -106,12 +104,20 @@ public class LlapDecider implements PhysicalPlanResolver { } private LlapMode mode; + private final LlapClusterStateForCompile clusterState; + + public LlapDecider(LlapClusterStateForCompile clusterState) { + this.clusterState = clusterState; + } + class LlapDecisionDispatcher implements Dispatcher { private final HiveConf conf; private final boolean doSkipUdfCheck; private final boolean arePermanentFnsAllowed; private final boolean shouldUber; + private final float minReducersPerExec; + private final int executorsPerNode; private List<MapJoinOperator> mapJoinOpList; private final Map<Rule, NodeProcessor> rules; @@ -121,6 +127,9 @@ public class LlapDecider implements PhysicalPlanResolver { arePermanentFnsAllowed = HiveConf.getBoolVar(conf, ConfVars.LLAP_ALLOW_PERMANENT_FNS); // Don't user uber in "all" mode - everything can go into LLAP, which is better than uber. shouldUber = HiveConf.getBoolVar(conf, ConfVars.LLAP_AUTO_ALLOW_UBER) && (mode != all); + minReducersPerExec = HiveConf.getFloatVar( + conf, ConfVars.TEZ_LLAP_MIN_REDUCER_PER_EXECUTOR); + executorsPerNode = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_NUM_EXECUTORS); // TODO# hmm mapJoinOpList = new ArrayList<MapJoinOperator>(); rules = getRules(); } @@ -139,22 +148,57 @@ public class LlapDecider implements PhysicalPlanResolver { return null; } - private void handleWork(TezWork tezWork, BaseWork work) - throws SemanticException { + private void handleWork(TezWork tezWork, BaseWork work) throws SemanticException { boolean workCanBeDoneInLlap = evaluateWork(tezWork, work); LOG.debug( "Work " + work + " " + (workCanBeDoneInLlap ? "can" : "cannot") + " be done in LLAP"); if (workCanBeDoneInLlap) { for (MapJoinOperator graceMapJoinOp : mapJoinOpList) { - LOG.debug( - "Disabling hybrid grace hash join in case of LLAP and non-dynamic partition hash join."); + LOG.debug("Disabling hybrid grace hash join in case of LLAP " + + "and non-dynamic partition hash join."); graceMapJoinOp.getConf().setHybridHashJoin(false); } + adjustAutoParallelism(work); + convertWork(tezWork, work); } mapJoinOpList.clear(); } + private void adjustAutoParallelism(BaseWork work) { + if (minReducersPerExec <= 0 || !(work instanceof ReduceWork)) return; + ReduceWork reduceWork = (ReduceWork)work; + if (reduceWork.isAutoReduceParallelism() == false && reduceWork.isUniformDistribution() == false) { + return; // Not based on ARP and cannot assume uniform distribution, bail. + } + clusterState.initClusterInfo(); + int targetCount = 0; + if (!clusterState.hasClusterInfo()) { + LOG.warn("Cannot determine LLAP cluster information"); + targetCount = (int)Math.ceil(minReducersPerExec * 1 * executorsPerNode); + } else { + targetCount = (int)Math.ceil(minReducersPerExec * (clusterState.getKnownExecutorCount() + + clusterState.getNodeCountWithUnknownExecutors() * executorsPerNode)); + } + // We only increase the targets here. + if (reduceWork.isAutoReduceParallelism()) { + int newMin = Math.max(reduceWork.getMinReduceTasks(), targetCount); + if (newMin < reduceWork.getMaxReduceTasks()) { + reduceWork.setMinReduceTasks(newMin); + reduceWork.getEdgePropRef().setAutoReduce(conf, true, newMin, + reduceWork.getMaxReduceTasks(), conf.getLongVar(HiveConf.ConfVars.BYTESPERREDUCER)); + } else { + reduceWork.setAutoReduceParallelism(false); + reduceWork.setNumReduceTasks(newMin); + // TODO: is this correct? based on the same logic as HIVE-14200 + reduceWork.getEdgePropRef().setAutoReduce(null, false, 0, 0, 0); + } + } else { + // UNIFORM || AUTOPARALLEL (maxed out) + reduceWork.setNumReduceTasks(Math.max(reduceWork.getNumReduceTasks(), targetCount)); + } + } + private void convertWork(TezWork tezWork, BaseWork work) throws SemanticException { http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java index 83e89af..a9c1e61 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.parse; import static org.apache.hadoop.hive.ql.plan.ReduceSinkDesc.ReducerTraits.AUTOPARALLEL; +import static org.apache.hadoop.hive.ql.plan.ReduceSinkDesc.ReducerTraits.UNIFORM; import java.util.*; @@ -96,6 +97,7 @@ public class GenTezUtils { reduceWork.setNumReduceTasks(reduceSink.getConf().getNumReducers()); reduceWork.setSlowStart(reduceSink.getConf().isSlowStart()); + reduceWork.setUniformDistribution(reduceSink.getConf().getReducerTraits().contains(UNIFORM)); if (isAutoReduceParallelism && reduceSink.getConf().getReducerTraits().contains(AUTOPARALLEL)) { @@ -103,6 +105,7 @@ public class GenTezUtils { final int maxReducers = context.conf.getIntVar(HiveConf.ConfVars.MAXREDUCERS); // estimated number of reducers final int nReducers = reduceSink.getConf().getNumReducers(); + // TODO# HERE // min we allow tez to pick int minPartition = Math.max(1, (int) (nReducers * minPartitionFactor)); @@ -139,6 +142,7 @@ public class GenTezUtils { edgeProp = new TezEdgeProperty(edgeType); edgeProp.setSlowStart(reduceWork.isSlowStart()); } + reduceWork.setEdgePropRef(edgeProp); tezWork.connect( context.preceedingWork, http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java index 26eda04..f87ca28 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hive.ql.parse; +import org.apache.hadoop.hive.ql.optimizer.physical.LlapClusterStateForCompile; + import com.google.common.base.Preconditions; import java.io.Serializable; import java.util.*; @@ -597,7 +599,8 @@ public class TezCompiler extends TaskCompiler { } if ("llap".equalsIgnoreCase(conf.getVar(HiveConf.ConfVars.HIVE_EXECUTION_MODE))) { - physicalCtx = new LlapDecider().resolve(physicalCtx); + LlapClusterStateForCompile llapInfo = LlapClusterStateForCompile.getClusterInfo(conf); + physicalCtx = new LlapDecider(llapInfo).resolve(physicalCtx); } else { LOG.debug("Skipping llap decider"); } http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java index 0cb1e57..50eeb99 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java @@ -85,6 +85,8 @@ public class ReduceWork extends BaseWork { // boolean that says whether tez auto reduce parallelism should be used private boolean isAutoReduceParallelism; + // boolean that says whether the data distribution is uniform hash (not java HashCode) + private transient boolean isUniformDistribution = false; // boolean that says whether to slow start or not private boolean isSlowStart = true; @@ -104,6 +106,8 @@ public class ReduceWork extends BaseWork { private String vectorReduceColumnSortOrder; private String vectorReduceColumnNullOrder; + private transient TezEdgeProperty edgeProp; + /** * If the plan has a reducer and correspondingly a reduce-sink, then store the TableDesc pointing * to keySerializeInfo of the ReduceSink @@ -231,6 +235,15 @@ public class ReduceWork extends BaseWork { this.isSlowStart = isSlowStart; } + // ReducerTraits.UNIFORM + public void setUniformDistribution(boolean isUniformDistribution) { + this.isUniformDistribution = isUniformDistribution; + } + + public boolean isUniformDistribution() { + return this.isUniformDistribution; + } + public void setMinReduceTasks(int minReduceTasks) { this.minReduceTasks = minReduceTasks; } @@ -355,4 +368,12 @@ public class ReduceWork extends BaseWork { } return new ReduceExplainVectorization(this); } + + public void setEdgePropRef(TezEdgeProperty edgeProp) { + this.edgeProp = edgeProp; + } + + public TezEdgeProperty getEdgePropRef() { + return edgeProp; + } } http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java index d87bee3..5d7ddc8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java @@ -50,6 +50,12 @@ public class TezEdgeProperty { public TezEdgeProperty(HiveConf hiveConf, EdgeType edgeType, boolean isAutoReduce, boolean isSlowStart, int minReducer, int maxReducer, long bytesPerReducer) { this(hiveConf, edgeType, -1); + setAutoReduce(hiveConf, isAutoReduce, minReducer, maxReducer, bytesPerReducer); + } + + public void setAutoReduce(HiveConf hiveConf, boolean isAutoReduce, int minReducer, + int maxReducer, long bytesPerReducer) { + this.hiveConf = hiveConf; this.minReducer = minReducer; this.maxReducer = maxReducer; this.isAutoReduce = isAutoReduce; http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/test/queries/clientpositive/subquery_scalar.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/subquery_scalar.q b/ql/src/test/queries/clientpositive/subquery_scalar.q index 99263bc..c7aeb11 100644 --- a/ql/src/test/queries/clientpositive/subquery_scalar.q +++ b/ql/src/test/queries/clientpositive/subquery_scalar.q @@ -1,5 +1,6 @@ set hive.mapred.mode=nonstrict; set hive.explain.user=false; +-- SORT_QUERY_RESULTS create table tnull(i int, c char(2)); insert into tnull values(NULL, NULL), (NULL, NULL); http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/test/queries/clientpositive/unionDistinct_1.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/unionDistinct_1.q b/ql/src/test/queries/clientpositive/unionDistinct_1.q index 9792267..f2a0dc1 100644 --- a/ql/src/test/queries/clientpositive/unionDistinct_1.q +++ b/ql/src/test/queries/clientpositive/unionDistinct_1.q @@ -1,8 +1,8 @@ set hive.mapred.mode=nonstrict; set hive.explain.user=false; --- union10.q +-- SORT_QUERY_RESULTS - +-- union10.q -- union case: all subqueries are a map-reduce jobs, 3 way union, same input for all sub-queries, followed by filesink http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/test/queries/clientpositive/vector_binary_join_groupby.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/vector_binary_join_groupby.q b/ql/src/test/queries/clientpositive/vector_binary_join_groupby.q index f203aef..826918f 100644 --- a/ql/src/test/queries/clientpositive/vector_binary_join_groupby.q +++ b/ql/src/test/queries/clientpositive/vector_binary_join_groupby.q @@ -42,20 +42,24 @@ STORED AS ORC; INSERT INTO TABLE hundredorc SELECT * FROM over1k LIMIT 100; EXPLAIN VECTORIZATION EXPRESSION -SELECT sum(hash(*)) -FROM hundredorc t1 JOIN hundredorc t2 ON t1.bin = t2.bin; +SELECT sum(hash(*)) k +FROM hundredorc t1 JOIN hundredorc t2 ON t1.bin = t2.bin +order by k; -SELECT sum(hash(*)) -FROM hundredorc t1 JOIN hundredorc t2 ON t1.bin = t2.bin; +SELECT sum(hash(*)) k +FROM hundredorc t1 JOIN hundredorc t2 ON t1.bin = t2.bin +order by k; EXPLAIN VECTORIZATION EXPRESSION SELECT count(*), bin FROM hundredorc -GROUP BY bin; +GROUP BY bin +order by bin; SELECT count(*), bin FROM hundredorc -GROUP BY bin; +GROUP BY bin +order by bin; -- HIVE-14045: Involve a binary vector scratch column for small table result (Native Vector MapJoin). http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/test/queries/clientpositive/vector_groupby_grouping_sets_limit.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/vector_groupby_grouping_sets_limit.q b/ql/src/test/queries/clientpositive/vector_groupby_grouping_sets_limit.q index fff706b..49cc4ef 100644 --- a/ql/src/test/queries/clientpositive/vector_groupby_grouping_sets_limit.q +++ b/ql/src/test/queries/clientpositive/vector_groupby_grouping_sets_limit.q @@ -3,6 +3,7 @@ SET hive.vectorized.execution.enabled=true; SET hive.vectorized.execution.reduce.enabled=true; set hive.fetch.task.conversion=none; set hive.cli.print.header=true; +-- SORT_QUERY_RESULTS CREATE TABLE T1_text(a STRING, b STRING, c STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY ' ' STORED AS TEXTFILE; @@ -13,31 +14,31 @@ CREATE TABLE T1 STORED AS ORC AS SELECT * FROM T1_text; -- SORT_QUERY_RESULTS EXPLAIN -SELECT a, b, count(*) from T1 group by a, b with cube LIMIT 10; +SELECT a, b, count(*) from T1 group by a, b with cube order by a, b LIMIT 10; -SELECT a, b, count(*) from T1 group by a, b with cube LIMIT 10; +SELECT a, b, count(*) from T1 group by a, b with cube order by a, b LIMIT 10; EXPLAIN -SELECT a, b, count(*) FROM T1 GROUP BY a, b GROUPING SETS (a, (a, b), b, ()) LIMIT 10; +SELECT a, b, count(*) FROM T1 GROUP BY a, b GROUPING SETS (a, (a, b), b, ()) order by a, b LIMIT 10; -SELECT a, b, count(*) FROM T1 GROUP BY a, b GROUPING SETS (a, (a, b), b, ()) LIMIT 10; +SELECT a, b, count(*) FROM T1 GROUP BY a, b GROUPING SETS (a, (a, b), b, ()) order by a, b LIMIT 10; EXPLAIN -SELECT a, b, count(*) FROM T1 GROUP BY a, b GROUPING SETS (a, (a, b)) LIMIT 10; +SELECT a, b, count(*) FROM T1 GROUP BY a, b GROUPING SETS (a, (a, b)) order by a, b LIMIT 10; -SELECT a, b, count(*) FROM T1 GROUP BY a, b GROUPING SETS (a, (a, b)) LIMIT 10; +SELECT a, b, count(*) FROM T1 GROUP BY a, b GROUPING SETS (a, (a, b)) order by a, b LIMIT 10; EXPLAIN -SELECT a FROM T1 GROUP BY a, b, c GROUPING SETS (a, b, c) LIMIT 10; +SELECT a FROM T1 GROUP BY a, b, c GROUPING SETS (a, b, c) order by a LIMIT 10; -SELECT a FROM T1 GROUP BY a, b, c GROUPING SETS (a, b, c) LIMIT 10; +SELECT a FROM T1 GROUP BY a, b, c GROUPING SETS (a, b, c) order by a LIMIT 10; EXPLAIN -SELECT a FROM T1 GROUP BY a GROUPING SETS ((a), (a)) LIMIT 10; +SELECT a FROM T1 GROUP BY a GROUPING SETS ((a), (a)) order by a LIMIT 10; -SELECT a FROM T1 GROUP BY a GROUPING SETS ((a), (a)) LIMIT 10; +SELECT a FROM T1 GROUP BY a GROUPING SETS ((a), (a)) order by a LIMIT 10; EXPLAIN -SELECT a + b, count(*) FROM T1 GROUP BY a + b GROUPING SETS (a+b) LIMIT 10; +SELECT a + b ab, count(*) FROM T1 GROUP BY a + b GROUPING SETS (a+b) order by ab LIMIT 10; -SELECT a + b, count(*) FROM T1 GROUP BY a + b GROUPING SETS (a+b) LIMIT 10; +SELECT a + b ab, count(*) FROM T1 GROUP BY a + b GROUPING SETS (a+b) order by ab LIMIT 10; http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/test/queries/clientpositive/vector_groupby_reduce.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/vector_groupby_reduce.q b/ql/src/test/queries/clientpositive/vector_groupby_reduce.q index bafb32e..f23b26f 100644 --- a/ql/src/test/queries/clientpositive/vector_groupby_reduce.q +++ b/ql/src/test/queries/clientpositive/vector_groupby_reduce.q @@ -98,6 +98,7 @@ select from store_sales group by ss_ticket_number +order by ss_ticket_number limit 20; select @@ -105,6 +106,7 @@ select from store_sales group by ss_ticket_number +order by ss_ticket_number limit 20; http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/test/results/clientpositive/llap/bucket_groupby.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/bucket_groupby.q.out b/ql/src/test/results/clientpositive/llap/bucket_groupby.q.out index 4ac5ac9..d724131 100644 --- a/ql/src/test/results/clientpositive/llap/bucket_groupby.q.out +++ b/ql/src/test/results/clientpositive/llap/bucket_groupby.q.out @@ -712,314 +712,314 @@ POSTHOOK: Input: default@clustergroupby POSTHOOK: Input: default@clustergroupby@ds=100 POSTHOOK: Input: default@clustergroupby@ds=101 #### A masked pattern was here #### -0 6 10 2 100 4 -103 4 104 4 -105 2 -11 2 111 2 -113 4 114 2 116 2 -118 4 -119 6 -12 4 -120 4 -125 4 126 2 128 6 -129 4 131 2 -133 2 134 4 -136 2 -137 4 -138 8 -143 2 -145 2 -146 4 -149 4 -15 4 -150 2 152 4 -153 2 155 2 -156 2 -157 2 -158 2 -160 2 -162 2 -163 2 -164 4 -165 4 -166 2 -167 6 -168 2 169 8 17 2 170 2 172 4 -174 4 -175 4 -176 4 -177 2 178 2 179 4 -18 4 -180 2 -181 2 -183 2 -186 2 -187 6 189 2 19 2 -190 2 -191 4 192 2 -193 6 194 2 195 4 -196 2 -197 4 199 6 -2 2 20 2 -200 4 -201 2 -202 2 -203 4 -205 4 207 4 -208 6 -209 4 213 4 -214 2 -216 4 -217 4 218 2 -219 4 -221 4 -222 2 223 4 -224 4 -226 2 -228 2 229 4 -230 10 233 4 235 2 -237 4 -238 4 -239 4 24 4 -241 2 -242 4 -244 2 -247 2 -248 2 -249 2 -252 2 -255 4 256 4 257 2 -258 2 26 4 -260 2 -262 2 -263 2 265 4 266 2 27 2 -272 4 -273 6 -274 2 -275 2 -277 8 278 4 -28 2 280 4 -281 4 -282 4 -283 2 284 2 -285 2 -286 2 -287 2 -288 4 -289 2 291 2 -292 2 -296 2 -298 6 30 2 302 2 305 2 -306 2 -307 4 -308 2 -309 4 -310 2 -311 6 -315 2 316 6 -317 4 -318 6 321 4 -322 4 323 2 -325 4 -327 6 33 2 -331 4 -332 2 333 4 -335 2 -336 2 338 2 -339 2 -34 2 -341 2 -342 4 344 4 -345 2 -348 10 -35 6 -351 2 353 4 356 2 -360 2 -362 2 364 2 -365 2 -366 2 -367 4 368 2 369 6 37 4 -373 2 -374 2 -375 2 377 2 378 2 -379 2 -382 4 -384 6 386 2 -389 2 392 2 -393 2 -394 2 -395 4 -396 6 397 4 -399 4 -4 2 -400 2 -401 10 -402 2 -403 6 404 4 -406 8 407 2 -409 6 -41 2 411 2 -413 4 -414 4 -417 6 418 2 419 2 -42 4 -421 2 424 4 -427 2 -429 4 -43 2 430 6 -431 6 432 2 -435 2 436 2 437 2 -438 6 -439 4 -44 2 -443 2 -444 2 -446 2 448 2 -449 2 -452 2 -453 2 454 6 -455 2 457 2 -458 4 -459 4 -460 2 -462 4 -463 4 -466 6 -467 2 468 8 -469 10 -47 2 -470 2 -472 2 -475 2 477 2 -478 4 479 2 -480 6 -481 2 -482 2 -483 2 -484 2 -485 2 -487 2 -489 8 490 2 -491 2 492 4 493 2 -494 2 495 2 496 2 497 2 -498 6 -5 6 -51 4 -53 2 -54 2 57 2 +67 4 +8 2 +80 2 +85 2 +9 2 +92 2 +97 4 +105 2 +11 2 +113 4 +136 2 +137 4 +138 8 +143 2 +150 2 +158 2 +160 2 +162 2 +165 4 +166 2 +174 4 +177 2 +18 4 +180 2 +186 2 +196 2 +197 4 +217 4 +219 4 +226 2 +230 10 +238 4 +242 4 +248 2 +252 2 +260 2 +263 2 +274 2 +277 8 +296 2 +307 4 +308 2 +325 4 +327 6 +342 4 +348 10 +35 6 +351 2 +367 4 +389 2 +393 2 +394 2 +4 2 +403 6 +409 6 +414 4 +421 2 +427 2 +429 4 +435 2 +438 6 +439 4 +455 2 +458 4 +460 2 +462 4 +475 2 +478 4 +480 6 +484 2 +487 2 +53 2 58 4 -64 2 -65 2 66 2 -67 4 69 2 -70 6 +77 2 +78 2 +83 4 +90 6 +95 4 +103 4 +120 4 +129 4 +133 2 +145 2 +146 4 +149 4 +15 4 +157 2 +181 2 +187 6 +191 4 +2 2 +200 4 +201 2 +202 2 +203 4 +208 6 +214 2 +216 4 +222 2 +224 4 +228 2 +237 4 +239 4 +241 2 +244 2 +255 4 +258 2 +262 2 +273 6 +275 2 +28 2 +282 4 +285 2 +286 2 +287 2 +288 4 +292 2 +298 6 +310 2 +311 6 +332 2 +335 2 +336 2 +339 2 +360 2 +362 2 +373 2 +384 6 +395 4 +396 6 +402 2 +417 6 +42 4 +43 2 +444 2 +449 2 +453 2 +459 4 +47 2 +481 2 +482 2 +483 2 +485 2 +489 8 +494 2 +65 2 72 4 74 2 76 4 -77 2 -78 2 -8 2 -80 2 82 2 -83 4 -84 4 -85 2 86 2 87 2 -9 2 -90 6 -92 2 -95 4 96 2 -97 4 +0 6 +118 4 +119 6 +12 4 +125 4 +153 2 +156 2 +163 2 +164 4 +167 6 +168 2 +175 4 +176 4 +183 2 +190 2 +193 6 +205 4 +209 4 +221 4 +247 2 +249 2 +272 4 +281 4 +283 2 +289 2 +306 2 +309 4 +315 2 +317 4 +318 6 +322 4 +331 4 +34 2 +341 2 +345 2 +365 2 +366 2 +374 2 +375 2 +379 2 +382 4 +399 4 +400 2 +401 10 +406 8 +41 2 +413 4 +431 6 +44 2 +443 2 +446 2 +452 2 +463 4 +466 6 +467 2 +469 10 +470 2 +472 2 +491 2 +498 6 +5 6 +51 4 +54 2 +64 2 +70 6 +84 4 98 4 PREHOOK: query: explain select key, count(1) from clustergroupby group by key, 3 http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/test/results/clientpositive/llap/cbo_rp_lineage2.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/cbo_rp_lineage2.q.out b/ql/src/test/results/clientpositive/llap/cbo_rp_lineage2.q.out index 4bc28ce..44f9d68 100644 --- a/ql/src/test/results/clientpositive/llap/cbo_rp_lineage2.q.out +++ b/ql/src/test/results/clientpositive/llap/cbo_rp_lineage2.q.out @@ -141,87 +141,87 @@ PREHOOK: type: QUERY PREHOOK: Input: default@dest1 #### A masked pattern was here #### {"version":"1.0","engine":"tez","database":"default","hash":"3901b5e3a164064736b3234355046340","queryText":"select key, count(1) a from dest1 group by key","edges":[],"vertices":[]} - 20 128 2 -146 2 -150 2 213 2 -224 2 +278 2 +369 2 + 20 +150 2 238 2 +66 2 +146 2 +224 2 255 2 273 2 -278 2 311 2 -369 2 401 2 406 2 -66 2 98 2 PREHOOK: query: select key k, count(*) from dest1 group by key PREHOOK: type: QUERY PREHOOK: Input: default@dest1 #### A masked pattern was here #### {"version":"1.0","engine":"tez","database":"default","hash":"0d5a212f10847aeaab31e8c31121e6d4","queryText":"select key k, count(*) from dest1 group by key","edges":[],"vertices":[]} - 20 128 2 -146 2 -150 2 213 2 -224 2 +278 2 +369 2 + 20 +150 2 238 2 +66 2 +146 2 +224 2 255 2 273 2 -278 2 311 2 -369 2 401 2 406 2 -66 2 98 2 PREHOOK: query: select key k, count(value) from dest1 group by key PREHOOK: type: QUERY PREHOOK: Input: default@dest1 #### A masked pattern was here #### {"version":"1.0","engine":"tez","database":"default","hash":"56429eccb04ded722f5bd9d9d8cf7260","queryText":"select key k, count(value) from dest1 group by key","edges":[],"vertices":[]} - 20 128 2 -146 2 -150 2 213 2 -224 2 +278 2 +369 2 + 20 +150 2 238 2 +66 2 +146 2 +224 2 255 2 273 2 -278 2 311 2 -369 2 401 2 406 2 -66 2 98 2 PREHOOK: query: select value, max(length(key)) from dest1 group by value PREHOOK: type: QUERY PREHOOK: Input: default@dest1 #### A masked pattern was here #### {"version":"1.0","engine":"tez","database":"default","hash":"7e1cfc3dece85b41b6f7c46365580cde","queryText":"select value, max(length(key)) from dest1 group by value","edges":[],"vertices":[]} - 3 val_146 3 -val_150 3 +val_27 0 +val_409 0 +val_484 0 +val_66 2 + 3 val_165 0 +val_238 3 val_193 0 val_213 3 -val_238 3 val_255 3 val_265 0 -val_27 0 val_273 3 val_278 3 -val_311 3 val_401 3 +val_150 3 +val_311 3 val_406 3 -val_409 0 -val_484 0 -val_66 2 val_98 2 PREHOOK: query: select value, max(length(key)) from dest1 group by value order by value limit 5 PREHOOK: type: QUERY http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/test/results/clientpositive/llap/cbo_rp_views.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/cbo_rp_views.q.out b/ql/src/test/results/clientpositive/llap/cbo_rp_views.q.out index 2ac2d3a..d603957 100644 --- a/ql/src/test/results/clientpositive/llap/cbo_rp_views.q.out +++ b/ql/src/test/results/clientpositive/llap/cbo_rp_views.q.out @@ -50,8 +50,8 @@ POSTHOOK: Input: default@cbo_t1 POSTHOOK: Input: default@cbo_t1@dt=2014 POSTHOOK: Input: default@v1 #### A masked pattern was here #### -NULL 1 +NULL 1 PREHOOK: query: select count(v1.c_int) from v1 join cbo_t2 on v1.c_int = cbo_t2.c_int PREHOOK: type: QUERY http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/test/results/clientpositive/llap/cbo_views.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/cbo_views.q.out b/ql/src/test/results/clientpositive/llap/cbo_views.q.out index 44e5501..25ea4cf 100644 --- a/ql/src/test/results/clientpositive/llap/cbo_views.q.out +++ b/ql/src/test/results/clientpositive/llap/cbo_views.q.out @@ -50,8 +50,8 @@ POSTHOOK: Input: default@cbo_t1 POSTHOOK: Input: default@cbo_t1@dt=2014 POSTHOOK: Input: default@v1 #### A masked pattern was here #### -NULL 1 +NULL 1 PREHOOK: query: select count(v1.c_int) from v1 join cbo_t2 on v1.c_int = cbo_t2.c_int PREHOOK: type: QUERY http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/test/results/clientpositive/llap/cluster.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/cluster.q.out b/ql/src/test/results/clientpositive/llap/cluster.q.out index 8c89ee3..2fa976b 100644 --- a/ql/src/test/results/clientpositive/llap/cluster.q.out +++ b/ql/src/test/results/clientpositive/llap/cluster.q.out @@ -1063,77 +1063,22 @@ CLUSTER BY unioninput.key POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### -0 val_0 -0 val_0 -0 val_0 10 val_10 -103 val_103 -103 val_103 104 val_104 104 val_104 -105 val_105 -11 val_11 111 val_111 -113 val_113 -113 val_113 114 val_114 116 val_116 -118 val_118 -118 val_118 -119 val_119 -119 val_119 -119 val_119 -12 val_12 -12 val_12 -120 val_120 -120 val_120 -125 val_125 -125 val_125 126 val_126 128 val_128 128 val_128 128 val_128 -129 val_129 -129 val_129 131 val_131 -133 val_133 134 val_134 134 val_134 -136 val_136 -137 val_137 -137 val_137 -138 val_138 -138 val_138 -138 val_138 -138 val_138 -143 val_143 -145 val_145 -146 val_146 -146 val_146 -149 val_149 -149 val_149 -15 val_15 -15 val_15 -150 val_150 152 val_152 152 val_152 -153 val_153 155 val_155 -156 val_156 -157 val_157 -158 val_158 -160 val_160 -162 val_162 -163 val_163 -164 val_164 -164 val_164 -165 val_165 -165 val_165 -166 val_166 -167 val_167 -167 val_167 -167 val_167 -168 val_168 169 val_169 169 val_169 169 val_169 @@ -1142,374 +1087,463 @@ POSTHOOK: Input: default@src 170 val_170 172 val_172 172 val_172 -174 val_174 -174 val_174 -175 val_175 -175 val_175 -176 val_176 -176 val_176 -177 val_177 178 val_178 179 val_179 179 val_179 -18 val_18 -18 val_18 -180 val_180 -181 val_181 -183 val_183 -186 val_186 -187 val_187 -187 val_187 -187 val_187 189 val_189 19 val_19 -190 val_190 -191 val_191 -191 val_191 192 val_192 -193 val_193 -193 val_193 -193 val_193 194 val_194 195 val_195 195 val_195 -196 val_196 -197 val_197 -197 val_197 199 val_199 199 val_199 199 val_199 -2 val_2 20 val_20 -200 val_200 -200 val_200 -201 val_201 -202 val_202 -203 val_203 -203 val_203 -205 val_205 -205 val_205 207 val_207 207 val_207 -208 val_208 -208 val_208 -208 val_208 -209 val_209 -209 val_209 213 val_213 213 val_213 -214 val_214 -216 val_216 -216 val_216 -217 val_217 -217 val_217 218 val_218 -219 val_219 -219 val_219 -221 val_221 -221 val_221 -222 val_222 223 val_223 223 val_223 -224 val_224 -224 val_224 -226 val_226 -228 val_228 229 val_229 229 val_229 -230 val_230 -230 val_230 -230 val_230 -230 val_230 -230 val_230 233 val_233 233 val_233 235 val_235 -237 val_237 -237 val_237 -238 val_238 -238 val_238 -239 val_239 -239 val_239 24 val_24 24 val_24 -241 val_241 -242 val_242 -242 val_242 -244 val_244 -247 val_247 -248 val_248 -249 val_249 -252 val_252 -255 val_255 -255 val_255 256 val_256 256 val_256 257 val_257 -258 val_258 26 val_26 26 val_26 -260 val_260 -262 val_262 -263 val_263 265 val_265 265 val_265 266 val_266 27 val_27 -272 val_272 -272 val_272 -273 val_273 -273 val_273 -273 val_273 -274 val_274 -275 val_275 -277 val_277 -277 val_277 -277 val_277 -277 val_277 278 val_278 278 val_278 -28 val_28 280 val_280 280 val_280 -281 val_281 -281 val_281 -282 val_282 -282 val_282 -283 val_283 284 val_284 -285 val_285 -286 val_286 -287 val_287 -288 val_288 -288 val_288 -289 val_289 291 val_291 -292 val_292 -296 val_296 -298 val_298 -298 val_298 -298 val_298 30 val_30 302 val_302 305 val_305 -306 val_306 -307 val_307 -307 val_307 -308 val_308 -309 val_309 -309 val_309 -310 val_310 -311 val_311 -311 val_311 -311 val_311 -315 val_315 316 val_316 316 val_316 316 val_316 -317 val_317 -317 val_317 -318 val_318 -318 val_318 -318 val_318 321 val_321 321 val_321 -322 val_322 -322 val_322 323 val_323 -325 val_325 -325 val_325 -327 val_327 -327 val_327 -327 val_327 33 val_33 -331 val_331 -331 val_331 -332 val_332 333 val_333 333 val_333 -335 val_335 -336 val_336 338 val_338 -339 val_339 -34 val_34 -341 val_341 -342 val_342 -342 val_342 344 val_344 344 val_344 -345 val_345 -348 val_348 -348 val_348 -348 val_348 -348 val_348 -348 val_348 -35 val_35 -35 val_35 -35 val_35 -351 val_351 353 val_353 353 val_353 356 val_356 -360 val_360 -362 val_362 364 val_364 -365 val_365 -366 val_366 -367 val_367 -367 val_367 368 val_368 369 val_369 369 val_369 369 val_369 37 val_37 37 val_37 -373 val_373 -374 val_374 -375 val_375 377 val_377 378 val_378 -379 val_379 -382 val_382 -382 val_382 -384 val_384 -384 val_384 -384 val_384 386 val_386 -389 val_389 392 val_392 -393 val_393 -394 val_394 -395 val_395 -395 val_395 -396 val_396 -396 val_396 -396 val_396 397 val_397 397 val_397 -399 val_399 -399 val_399 -4 val_4 -400 val_400 -401 val_401 -401 val_401 -401 val_401 -401 val_401 -401 val_401 -402 val_402 -403 val_403 -403 val_403 -403 val_403 404 val_404 404 val_404 -406 val_406 -406 val_406 -406 val_406 -406 val_406 407 val_407 -409 val_409 -409 val_409 -409 val_409 -41 val_41 411 val_411 -413 val_413 -413 val_413 -414 val_414 -414 val_414 -417 val_417 -417 val_417 -417 val_417 418 val_418 419 val_419 -42 val_42 -42 val_42 -421 val_421 424 val_424 424 val_424 -427 val_427 -429 val_429 -429 val_429 -43 val_43 430 val_430 430 val_430 430 val_430 -431 val_431 -431 val_431 -431 val_431 432 val_432 -435 val_435 436 val_436 437 val_437 +448 val_448 +454 val_454 +454 val_454 +454 val_454 +457 val_457 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +477 val_477 +479 val_479 +490 val_490 +492 val_492 +492 val_492 +493 val_493 +495 val_495 +496 val_496 +497 val_497 +57 val_57 +67 val_67 +67 val_67 +8 val_8 +80 val_80 +85 val_85 +9 val_9 +92 val_92 +97 val_97 +97 val_97 +105 val_105 +11 val_11 +113 val_113 +113 val_113 +136 val_136 +137 val_137 +137 val_137 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +143 val_143 +150 val_150 +158 val_158 +160 val_160 +162 val_162 +165 val_165 +165 val_165 +166 val_166 +174 val_174 +174 val_174 +177 val_177 +18 val_18 +18 val_18 +180 val_180 +186 val_186 +196 val_196 +197 val_197 +197 val_197 +217 val_217 +217 val_217 +219 val_219 +219 val_219 +226 val_226 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +238 val_238 +238 val_238 +242 val_242 +242 val_242 +248 val_248 +252 val_252 +260 val_260 +263 val_263 +274 val_274 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +296 val_296 +307 val_307 +307 val_307 +308 val_308 +325 val_325 +325 val_325 +327 val_327 +327 val_327 +327 val_327 +342 val_342 +342 val_342 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +35 val_35 +35 val_35 +35 val_35 +351 val_351 +367 val_367 +367 val_367 +389 val_389 +393 val_393 +394 val_394 +4 val_4 +403 val_403 +403 val_403 +403 val_403 +409 val_409 +409 val_409 +409 val_409 +414 val_414 +414 val_414 +421 val_421 +427 val_427 +429 val_429 +429 val_429 +435 val_435 438 val_438 438 val_438 438 val_438 439 val_439 439 val_439 -44 val_44 -443 val_443 -444 val_444 -446 val_446 -448 val_448 -449 val_449 -452 val_452 -453 val_453 -454 val_454 -454 val_454 -454 val_454 455 val_455 -457 val_457 458 val_458 458 val_458 -459 val_459 -459 val_459 460 val_460 462 val_462 462 val_462 -463 val_463 -463 val_463 -466 val_466 -466 val_466 -466 val_466 -467 val_467 -468 val_468 -468 val_468 -468 val_468 -468 val_468 -469 val_469 -469 val_469 -469 val_469 -469 val_469 -469 val_469 -47 val_47 -470 val_470 -472 val_472 475 val_475 -477 val_477 478 val_478 478 val_478 -479 val_479 480 val_480 480 val_480 480 val_480 +484 val_484 +487 val_487 +53 val_53 +58 val_58 +58 val_58 +66 val_66 +69 val_69 +77 val_77 +78 val_78 +83 val_83 +83 val_83 +90 val_90 +90 val_90 +90 val_90 +95 val_95 +95 val_95 +103 val_103 +103 val_103 +120 val_120 +120 val_120 +129 val_129 +129 val_129 +133 val_133 +145 val_145 +146 val_146 +146 val_146 +149 val_149 +149 val_149 +15 val_15 +15 val_15 +157 val_157 +181 val_181 +187 val_187 +187 val_187 +187 val_187 +191 val_191 +191 val_191 +2 val_2 +200 val_200 +200 val_200 +201 val_201 +202 val_202 +203 val_203 +203 val_203 +208 val_208 +208 val_208 +208 val_208 +214 val_214 +216 val_216 +216 val_216 +222 val_222 +224 val_224 +224 val_224 +228 val_228 +237 val_237 +237 val_237 +239 val_239 +239 val_239 +241 val_241 +244 val_244 +255 val_255 +255 val_255 +258 val_258 +262 val_262 +273 val_273 +273 val_273 +273 val_273 +275 val_275 +28 val_28 +282 val_282 +282 val_282 +285 val_285 +286 val_286 +287 val_287 +288 val_288 +288 val_288 +292 val_292 +298 val_298 +298 val_298 +298 val_298 +310 val_310 +311 val_311 +311 val_311 +311 val_311 +332 val_332 +335 val_335 +336 val_336 +339 val_339 +360 val_360 +362 val_362 +373 val_373 +384 val_384 +384 val_384 +384 val_384 +395 val_395 +395 val_395 +396 val_396 +396 val_396 +396 val_396 +402 val_402 +417 val_417 +417 val_417 +417 val_417 +42 val_42 +42 val_42 +43 val_43 +444 val_444 +449 val_449 +453 val_453 +459 val_459 +459 val_459 +47 val_47 481 val_481 482 val_482 483 val_483 -484 val_484 485 val_485 -487 val_487 489 val_489 489 val_489 489 val_489 489 val_489 -490 val_490 -491 val_491 -492 val_492 -492 val_492 -493 val_493 494 val_494 -495 val_495 -496 val_496 -497 val_497 +65 val_65 +72 val_72 +72 val_72 +74 val_74 +76 val_76 +76 val_76 +82 val_82 +86 val_86 +87 val_87 +96 val_96 +0 val_0 +0 val_0 +0 val_0 +118 val_118 +118 val_118 +119 val_119 +119 val_119 +119 val_119 +12 val_12 +12 val_12 +125 val_125 +125 val_125 +153 val_153 +156 val_156 +163 val_163 +164 val_164 +164 val_164 +167 val_167 +167 val_167 +167 val_167 +168 val_168 +175 val_175 +175 val_175 +176 val_176 +176 val_176 +183 val_183 +190 val_190 +193 val_193 +193 val_193 +193 val_193 +205 val_205 +205 val_205 +209 val_209 +209 val_209 +221 val_221 +221 val_221 +247 val_247 +249 val_249 +272 val_272 +272 val_272 +281 val_281 +281 val_281 +283 val_283 +289 val_289 +306 val_306 +309 val_309 +309 val_309 +315 val_315 +317 val_317 +317 val_317 +318 val_318 +318 val_318 +318 val_318 +322 val_322 +322 val_322 +331 val_331 +331 val_331 +34 val_34 +341 val_341 +345 val_345 +365 val_365 +366 val_366 +374 val_374 +375 val_375 +379 val_379 +382 val_382 +382 val_382 +399 val_399 +399 val_399 +400 val_400 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +41 val_41 +413 val_413 +413 val_413 +431 val_431 +431 val_431 +431 val_431 +44 val_44 +443 val_443 +446 val_446 +452 val_452 +463 val_463 +463 val_463 +466 val_466 +466 val_466 +466 val_466 +467 val_467 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +470 val_470 +472 val_472 +491 val_491 498 val_498 498 val_498 498 val_498 @@ -1518,46 +1552,12 @@ POSTHOOK: Input: default@src 5 val_5 51 val_51 51 val_51 -53 val_53 54 val_54 -57 val_57 -58 val_58 -58 val_58 64 val_64 -65 val_65 -66 val_66 -67 val_67 -67 val_67 -69 val_69 70 val_70 70 val_70 70 val_70 -72 val_72 -72 val_72 -74 val_74 -76 val_76 -76 val_76 -77 val_77 -78 val_78 -8 val_8 -80 val_80 -82 val_82 -83 val_83 -83 val_83 84 val_84 84 val_84 -85 val_85 -86 val_86 -87 val_87 -9 val_9 -90 val_90 -90 val_90 -90 val_90 -92 val_92 -95 val_95 -95 val_95 -96 val_96 -97 val_97 -97 val_97 98 val_98 98 val_98 http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/test/results/clientpositive/llap/correlationoptimizer1.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/correlationoptimizer1.q.out b/ql/src/test/results/clientpositive/llap/correlationoptimizer1.q.out index 86d94b4..cc86d15 100644 --- a/ql/src/test/results/clientpositive/llap/correlationoptimizer1.q.out +++ b/ql/src/test/results/clientpositive/llap/correlationoptimizer1.q.out @@ -1438,20 +1438,20 @@ POSTHOOK: Input: default@src POSTHOOK: Input: default@src1 #### A masked pattern was here #### NULL 10 -128 NULL 1 -146 val_146 2 -150 val_150 1 213 val_213 2 -224 NULL 1 238 val_238 2 +406 val_406 4 +66 val_66 1 +224 NULL 1 +278 val_278 2 +128 NULL 1 +150 val_150 1 255 val_255 2 273 val_273 3 -278 val_278 2 +401 val_401 5 +146 val_146 2 311 val_311 3 369 NULL 1 -401 val_401 5 -406 val_406 4 -66 val_66 1 98 val_98 2 PREHOOK: query: EXPLAIN SELECT x.key, y.value, count(1) AS cnt @@ -1573,20 +1573,20 @@ POSTHOOK: Input: default@src POSTHOOK: Input: default@src1 #### A masked pattern was here #### NULL 10 -128 NULL 1 -146 val_146 2 -150 val_150 1 213 val_213 2 -224 NULL 1 238 val_238 2 +406 val_406 4 +66 val_66 1 +224 NULL 1 +278 val_278 2 +128 NULL 1 +150 val_150 1 255 val_255 2 273 val_273 3 -278 val_278 2 +401 val_401 5 +146 val_146 2 311 val_311 3 369 NULL 1 -401 val_401 5 -406 val_406 4 -66 val_66 1 98 val_98 2 PREHOOK: query: EXPLAIN SELECT SUM(HASH(tmp.key)), SUM(HASH(tmp.cnt))
