Repository: hive Updated Branches: refs/heads/spark 725248174 -> a8c49ef41
HIVE-9139: Clean up GenSparkProcContext.clonedReduceSinks and related code [Spark Branch] (Chao Sun, reviewed by Xuefu Zhang) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a8c49ef4 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a8c49ef4 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a8c49ef4 Branch: refs/heads/spark Commit: a8c49ef419371cd827115f39a2a7b75c544e4eae Parents: 7252481 Author: Chao Sun <sunc...@apache.org> Authored: Mon Aug 17 10:40:53 2015 -0700 Committer: Chao Sun <sunc...@apache.org> Committed: Mon Aug 17 10:40:53 2015 -0700 ---------------------------------------------------------------------- .../hadoop/hive/ql/parse/spark/GenSparkProcContext.java | 2 -- .../org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java | 7 ------- 2 files changed, 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/a8c49ef4/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java index 0a0c791..62237e1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java @@ -129,7 +129,6 @@ public class GenSparkProcContext implements NodeProcessorCtx { public final Map<Operator<?>, BaseWork> unionWorkMap; public final List<UnionOperator> currentUnionOperators; public final Set<BaseWork> workWithUnionOperators; - public final Set<ReduceSinkOperator> clonedReduceSinks; public final Set<FileSinkOperator> fileSinkSet; public final Map<FileSinkOperator, List<FileSinkOperator>> fileSinkMap; @@ -180,7 +179,6 @@ public class GenSparkProcContext implements NodeProcessorCtx { this.unionWorkMap = new LinkedHashMap<Operator<?>, BaseWork>(); this.currentUnionOperators = new LinkedList<UnionOperator>(); this.workWithUnionOperators = new LinkedHashSet<BaseWork>(); - this.clonedReduceSinks = new LinkedHashSet<ReduceSinkOperator>(); this.fileSinkSet = new LinkedHashSet<FileSinkOperator>(); this.fileSinkMap = new LinkedHashMap<FileSinkOperator, List<FileSinkOperator>>(); this.pruningSinkSet = new LinkedHashSet<Operator<?>>(); http://git-wip-us.apache.org/repos/asf/hive/blob/a8c49ef4/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java index 3dd6d92..2d5bb62 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java @@ -94,12 +94,6 @@ public class GenSparkWork implements NodeProcessor { LOG.debug("Root operator: " + root); LOG.debug("Leaf operator: " + operator); - if (context.clonedReduceSinks.contains(operator)) { - // if we're visiting a terminal we've created ourselves, - // just skip and keep going - return null; - } - SparkWork sparkWork = context.currentTask.getWork(); SMBMapJoinOperator smbOp = GenSparkUtils.getChildOperator(root, SMBMapJoinOperator.class); @@ -192,7 +186,6 @@ public class GenSparkWork implements NodeProcessor { // we've already set this one up. Need to clone for the next work. r = (ReduceSinkOperator) OperatorFactory.getAndMakeChild( (ReduceSinkDesc)r.getConf().clone(), r.getParentOperators()); - context.clonedReduceSinks.add(r); } r.getConf().setOutputName(work.getName()); }