Repository: hive Updated Branches: refs/heads/master 7e0b08c1d -> 12cb6a98e
HIVE-13417: Some vector operators return 'OP' as name (Gunther Hagleitner, reviewed by Vikram Dixit K) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/12cb6a98 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/12cb6a98 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/12cb6a98 Branch: refs/heads/master Commit: 12cb6a98e20fff9a1e0d8c615c76af2dc85ebce4 Parents: 7e0b08c Author: Gunther Hagleitner <gunt...@apache.org> Authored: Mon Apr 4 10:51:09 2016 -0700 Committer: Gunther Hagleitner <gunt...@apache.org> Committed: Thu Apr 7 14:44:04 2016 -0700 ---------------------------------------------------------------------- .../hive/ql/exec/AbstractFileMergeOperator.java | 9 ++ .../hive/ql/exec/AppMasterEventOperator.java | 4 +- .../hadoop/hive/ql/exec/CollectOperator.java | 11 ++- .../hadoop/hive/ql/exec/CommonJoinOperator.java | 8 +- .../hadoop/hive/ql/exec/DemuxOperator.java | 8 +- .../hadoop/hive/ql/exec/DummyStoreOperator.java | 11 ++- .../hadoop/hive/ql/exec/FilterOperator.java | 4 +- .../hadoop/hive/ql/exec/ForwardOperator.java | 4 +- .../hadoop/hive/ql/exec/GroupByOperator.java | 4 +- .../hive/ql/exec/HashTableDummyOperator.java | 4 +- .../ql/exec/LateralViewForwardOperator.java | 5 +- .../hive/ql/exec/LateralViewJoinOperator.java | 4 +- .../hadoop/hive/ql/exec/LimitOperator.java | 4 +- .../hadoop/hive/ql/exec/ListSinkOperator.java | 11 ++- .../apache/hadoop/hive/ql/exec/MapOperator.java | 9 +- .../apache/hadoop/hive/ql/exec/MuxOperator.java | 8 +- .../apache/hadoop/hive/ql/exec/Operator.java | 22 +++-- .../apache/hadoop/hive/ql/exec/PTFOperator.java | 4 +- .../hadoop/hive/ql/exec/ScriptOperator.java | 40 ++++---- .../hadoop/hive/ql/exec/SelectOperator.java | 4 +- .../ql/exec/SparkHashTableSinkOperator.java | 10 +- .../hadoop/hive/ql/exec/TableScanOperator.java | 2 +- .../hadoop/hive/ql/exec/TerminalOperator.java | 10 ++ .../hadoop/hive/ql/exec/UDTFOperator.java | 8 +- .../hadoop/hive/ql/exec/UnionOperator.java | 2 +- .../vector/VectorAppMasterEventOperator.java | 11 +-- .../ql/exec/vector/VectorFileSinkOperator.java | 8 +- .../ql/exec/vector/VectorFilterOperator.java | 8 +- .../ql/exec/vector/VectorGroupByOperator.java | 25 +++-- .../ql/exec/vector/VectorLimitOperator.java | 5 +- .../ql/exec/vector/VectorMapJoinOperator.java | 11 ++- .../VectorMapJoinOuterFilteredOperator.java | 8 +- .../hive/ql/exec/vector/VectorMapOperator.java | 5 +- .../exec/vector/VectorReduceSinkOperator.java | 8 +- .../exec/vector/VectorSMBMapJoinOperator.java | 13 +-- .../ql/exec/vector/VectorSelectOperator.java | 21 +++-- .../VectorSparkHashTableSinkOperator.java | 6 +- ...VectorSparkPartitionPruningSinkOperator.java | 8 +- .../hive/ql/optimizer/physical/LlapDecider.java | 12 ++- .../SparkPartitionPruningSinkOperator.java | 16 ++-- .../hadoop/hive/ql/exec/TestOperatorNames.java | 98 ++++++++++++++++++++ .../vector/util/FakeCaptureOutputOperator.java | 10 +- .../util/FakeVectorDataSourceOperator.java | 11 ++- .../clientpositive/explain_logical.q.out | 6 +- .../clientpositive/tez/explainuser_3.q.out | 8 +- .../tez/vector_aggregate_without_gby.q.out | 6 +- .../tez/vector_auto_smb_mapjoin_14.q.out | 38 ++++---- .../tez/vector_groupby_mapjoin.q.out | 4 +- .../tez/vector_join_part_col_char.q.out | 10 +- .../tez/vectorized_parquet_types.q.out | 2 +- .../tez/vectorized_timestamp.q.out | 8 +- 51 files changed, 360 insertions(+), 216 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractFileMergeOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractFileMergeOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractFileMergeOperator.java index 154a78b..dfad6c1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractFileMergeOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractFileMergeOperator.java @@ -281,4 +281,13 @@ public abstract class AbstractFileMergeOperator<T extends FileMergeDesc> return null; } } + + @Override + public String getName() { + return AbstractFileMergeOperator.getOperatorName(); + } + + public static String getOperatorName() { + return "MERGE"; + } } http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/AppMasterEventOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/AppMasterEventOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/AppMasterEventOperator.java index 743098b..bf30ef1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/AppMasterEventOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/AppMasterEventOperator.java @@ -20,9 +20,7 @@ package org.apache.hadoop.hive.ql.exec; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.Collection; import java.util.Collections; -import java.util.concurrent.Future; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; @@ -151,7 +149,7 @@ public class AppMasterEventOperator extends Operator<AppMasterEventDesc> { */ @Override public String getName() { - return getOperatorName(); + return AppMasterEventOperator.getOperatorName(); } static public String getOperatorName() { http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/CollectOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/CollectOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/CollectOperator.java index 27ddf13..16675f2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/CollectOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/CollectOperator.java @@ -20,8 +20,6 @@ package org.apache.hadoop.hive.ql.exec; import java.io.Serializable; import java.util.ArrayList; -import java.util.Collection; -import java.util.concurrent.Future; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.CompilationOpContext; @@ -94,4 +92,13 @@ public class CollectOperator extends Operator<CollectDesc> implements public OperatorType getType() { return null; } + + @Override + public String getName() { + return CollectOperator.getOperatorName(); + } + + public static String getOperatorName() { + return "COLLECT"; + } } http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java index f8520f8..117a81e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java @@ -21,14 +21,10 @@ package org.apache.hadoop.hive.ql.exec; import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.Future; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.CompilationOpContext; @@ -44,6 +40,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Join operator implementation. @@ -793,7 +791,7 @@ public abstract class CommonJoinOperator<T extends JoinDesc> extends @Override public String getName() { - return getOperatorName(); + return CommonJoinOperator.getOperatorName(); } static public String getOperatorName() { http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/DemuxOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DemuxOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DemuxOperator.java index b897c16..c184742 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DemuxOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DemuxOperator.java @@ -21,14 +21,10 @@ package org.apache.hadoop.hive.ql.exec; import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.concurrent.Future; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -41,6 +37,8 @@ import org.apache.hadoop.hive.serde2.SerDeUtils; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hive.common.util.ReflectionUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * DemuxOperator is an operator used by MapReduce Jobs optimized by @@ -374,7 +372,7 @@ public class DemuxOperator extends Operator<DemuxDesc> */ @Override public String getName() { - return getOperatorName(); + return DemuxOperator.getOperatorName(); } static public String getOperatorName() { http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/DummyStoreOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DummyStoreOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DummyStoreOperator.java index 06a3884..2a1be63 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DummyStoreOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DummyStoreOperator.java @@ -19,8 +19,6 @@ package org.apache.hadoop.hive.ql.exec; import java.io.Serializable; -import java.util.Collection; -import java.util.concurrent.Future; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.CompilationOpContext; @@ -118,4 +116,13 @@ public class DummyStoreOperator extends Operator<DummyStoreDesc> implements Seri public OperatorType getType() { return OperatorType.FORWARD; } + + @Override + public String getName() { + return DummyStoreOperator.getOperatorName(); + } + + public static String getOperatorName() { + return "DUMMY_STORE"; + } } http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java index 08f2633..bd0d28c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java @@ -19,8 +19,6 @@ package org.apache.hadoop.hive.ql.exec; import java.io.Serializable; -import java.util.Collection; -import java.util.concurrent.Future; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; @@ -134,7 +132,7 @@ public class FilterOperator extends Operator<FilterDesc> implements */ @Override public String getName() { - return getOperatorName(); + return FilterOperator.getOperatorName(); } static public String getOperatorName() { http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/ForwardOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ForwardOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ForwardOperator.java index 2df7cca..8e516ce 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ForwardOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ForwardOperator.java @@ -19,8 +19,6 @@ package org.apache.hadoop.hive.ql.exec; import java.io.Serializable; -import java.util.Collection; -import java.util.concurrent.Future; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.CompilationOpContext; @@ -55,7 +53,7 @@ public class ForwardOperator extends Operator<ForwardDesc> implements */ @Override public String getName() { - return getOperatorName(); + return ForwardOperator.getOperatorName(); } static public String getOperatorName() { http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java index 0839b42..e39b75e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java @@ -24,14 +24,12 @@ import java.lang.reflect.Field; import java.sql.Timestamp; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.Future; import javolution.util.FastBitSet; @@ -1131,7 +1129,7 @@ public class GroupByOperator extends Operator<GroupByDesc> { */ @Override public String getName() { - return getOperatorName(); + return GroupByOperator.getOperatorName(); } static public String getOperatorName() { http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableDummyOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableDummyOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableDummyOperator.java index 4749247..0aab7a8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableDummyOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableDummyOperator.java @@ -18,8 +18,6 @@ package org.apache.hadoop.hive.ql.exec; import java.io.Serializable; -import java.util.Collection; -import java.util.concurrent.Future; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.CompilationOpContext; @@ -67,7 +65,7 @@ public class HashTableDummyOperator extends Operator<HashTableDummyDesc> impleme @Override public String getName() { - return getOperatorName(); + return HashTableDummyOperator.getOperatorName(); } static public String getOperatorName() { http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/LateralViewForwardOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/LateralViewForwardOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/LateralViewForwardOperator.java index 4c94ad9..edc400a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/LateralViewForwardOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/LateralViewForwardOperator.java @@ -18,9 +18,6 @@ package org.apache.hadoop.hive.ql.exec; -import java.util.Collection; -import java.util.concurrent.Future; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -44,7 +41,7 @@ public class LateralViewForwardOperator extends Operator<LateralViewForwardDesc> @Override public String getName() { - return getOperatorName(); + return LateralViewForwardOperator.getOperatorName(); } static public String getOperatorName() { http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/LateralViewJoinOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/LateralViewJoinOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/LateralViewJoinOperator.java index 7407dc6..cf3c5f0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/LateralViewJoinOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/LateralViewJoinOperator.java @@ -19,9 +19,7 @@ package org.apache.hadoop.hive.ql.exec; import java.util.ArrayList; -import java.util.Collection; import java.util.List; -import java.util.concurrent.Future; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.CompilationOpContext; @@ -149,7 +147,7 @@ public class LateralViewJoinOperator extends Operator<LateralViewJoinDesc> { @Override public String getName() { - return getOperatorName(); + return LateralViewJoinOperator.getOperatorName(); } static public String getOperatorName() { http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/LimitOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/LimitOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/LimitOperator.java index 239d56b..9676d70 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/LimitOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/LimitOperator.java @@ -19,8 +19,6 @@ package org.apache.hadoop.hive.ql.exec; import java.io.Serializable; -import java.util.Collection; -import java.util.concurrent.Future; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.CompilationOpContext; @@ -73,7 +71,7 @@ public class LimitOperator extends Operator<LimitDesc> implements Serializable { @Override public String getName() { - return getOperatorName(); + return LimitOperator.getOperatorName(); } static public String getOperatorName() { http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/ListSinkOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ListSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ListSinkOperator.java index 2f2abc1..b081cd0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ListSinkOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ListSinkOperator.java @@ -18,10 +18,8 @@ package org.apache.hadoop.hive.ql.exec; -import java.util.Collection; import java.util.List; import java.util.Properties; -import java.util.concurrent.Future; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.CompilationOpContext; @@ -107,4 +105,13 @@ public class ListSinkOperator extends Operator<ListSinkDesc> { public OperatorType getType() { return OperatorType.FORWARD; } + + @Override + public String getName() { + return ListSinkOperator.getOperatorName(); + } + + public static String getOperatorName() { + return "LIST_SINK"; + } } http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java index 4608f70..b1f9958 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java @@ -21,7 +21,6 @@ package org.apache.hadoop.hive.ql.exec; import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; @@ -31,18 +30,15 @@ import java.util.Map.Entry; import java.util.Properties; import java.util.Set; import java.util.TreeMap; -import java.util.concurrent.Future; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.ql.CompilationOpContext; -import org.apache.hadoop.hive.ql.exec.MapOperator.MapOpCtx; import org.apache.hadoop.hive.ql.exec.mr.ExecMapperContext; import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.io.RecordIdentifier; -import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.VirtualColumn; import org.apache.hadoop.hive.ql.plan.MapWork; @@ -59,14 +55,12 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; -import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.util.StringUtils; @@ -681,7 +675,7 @@ public class MapOperator extends Operator<MapWork> implements Serializable, Clon @Override public String getName() { - return getOperatorName(); + return MapOperator.getOperatorName(); } static public String getOperatorName() { @@ -716,4 +710,5 @@ public class MapOperator extends Operator<MapWork> implements Serializable, Clon public Map<Integer, DummyStoreOperator> getConnectedOperators() { return connectedOperators; } + } http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/MuxOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MuxOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MuxOperator.java index d8444fb..9849243 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MuxOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MuxOperator.java @@ -21,12 +21,8 @@ package org.apache.hadoop.hive.ql.exec; import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.List; -import java.util.concurrent.Future; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -36,6 +32,8 @@ import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.api.OperatorType; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * MuxOperator is used in the Reduce side of MapReduce jobs optimized by Correlation Optimizer. @@ -331,7 +329,7 @@ public class MuxOperator extends Operator<MuxDesc> implements Serializable{ */ @Override public String getName() { - return getOperatorName(); + return MuxOperator.getOperatorName(); } static public String getOperatorName() { http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java index 571620e..f330564 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java @@ -31,8 +31,6 @@ import java.util.concurrent.CancellationException; import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicBoolean; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.mr.ExecMapperContext; @@ -51,6 +49,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.mapred.OutputCollector; import org.apache.hadoop.mapred.Reporter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Base operator implementation. @@ -887,15 +887,8 @@ public abstract class Operator<T extends OperatorDesc> implements Serializable,C } } - /** - * Implements the getName function for the Node Interface. - * - * @return the name of the operator - */ @Override - public String getName() { - return getOperatorName(); - } + public abstract String getName(); static public String getOperatorName() { return "OP"; @@ -1351,6 +1344,15 @@ public abstract class Operator<T extends OperatorDesc> implements Serializable,C } @Override + public String getName() { + return DummyOperator.getOperatorName(); + } + + public static String getOperatorName() { + return "DUMMY"; + } + + @Override protected void initializeOp(Configuration conf) { } } http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java index 2e9e539..37ae8fe 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java @@ -19,11 +19,9 @@ package org.apache.hadoop.hive.ql.exec; import java.io.Serializable; -import java.util.Collection; import java.util.Iterator; import java.util.List; import java.util.Stack; -import java.util.concurrent.Future; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.CompilationOpContext; @@ -184,7 +182,7 @@ public class PTFOperator extends Operator<PTFDesc> implements Serializable { */ @Override public String getName() { - return getOperatorName(); + return PTFOperator.getOperatorName(); } static public String getOperatorName() { http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java index 2bce5d0..00884cd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java @@ -18,6 +18,24 @@ package org.apache.hadoop.hive.ql.exec; +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.TimeUnit; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.CompilationOpContext; @@ -41,26 +59,6 @@ import org.apache.spark.SparkConf; import org.apache.spark.SparkEnv; import org.apache.spark.SparkFiles; -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.File; -import java.io.IOException; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Map; -import java.util.Set; -import java.util.Timer; -import java.util.TimerTask; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; - /** * ScriptOperator. * @@ -862,7 +860,7 @@ public class ScriptOperator extends Operator<ScriptDesc> implements @Override public String getName() { - return getOperatorName(); + return ScriptOperator.getOperatorName(); } static public String getOperatorName() { http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java index e7c23e8..9049ddd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java @@ -19,9 +19,7 @@ package org.apache.hadoop.hive.ql.exec; import java.io.Serializable; -import java.util.Collection; import java.util.List; -import java.util.concurrent.Future; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; @@ -102,7 +100,7 @@ public class SelectOperator extends Operator<SelectDesc> implements Serializable */ @Override public String getName() { - return getOperatorName(); + return SelectOperator.getOperatorName(); } static public String getOperatorName() { http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/SparkHashTableSinkOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/SparkHashTableSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/SparkHashTableSinkOperator.java index 5837614..523ff7c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/SparkHashTableSinkOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/SparkHashTableSinkOperator.java @@ -21,13 +21,9 @@ import java.io.BufferedOutputStream; import java.io.ObjectOutputStream; import java.io.OutputStream; import java.io.Serializable; -import java.util.Collection; import java.util.Set; -import java.util.concurrent.Future; import org.apache.commons.io.FileExistsException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -44,6 +40,8 @@ import org.apache.hadoop.hive.ql.plan.SparkHashTableSinkDesc; import org.apache.hadoop.hive.ql.plan.api.OperatorType; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class SparkHashTableSinkOperator extends TerminalOperator<SparkHashTableSinkDesc> implements Serializable { @@ -195,6 +193,10 @@ public class SparkHashTableSinkOperator */ @Override public String getName() { + return SparkHashTableSinkOperator.getOperatorName(); + } + + public static String getOperatorName() { return HashTableSinkOperator.getOperatorName(); } http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java index 5f2a0c2..6afe957 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java @@ -277,7 +277,7 @@ public class TableScanOperator extends Operator<TableScanDesc> implements **/ @Override public String getName() { - return getOperatorName(); + return TableScanOperator.getOperatorName(); } static public String getOperatorName() { http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/TerminalOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/TerminalOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/TerminalOperator.java index 04d6c9f..aec2f11 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/TerminalOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/TerminalOperator.java @@ -38,4 +38,14 @@ public abstract class TerminalOperator<T extends OperatorDesc> extends public TerminalOperator(CompilationOpContext ctx) { super(ctx); } + + @Override + public String getName() { + return getOperatorName(); + } + + static public String getOperatorName() { + return "END"; + } + } http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/UDTFOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/UDTFOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/UDTFOperator.java index 1dae963..a75b52a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/UDTFOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/UDTFOperator.java @@ -20,13 +20,9 @@ package org.apache.hadoop.hive.ql.exec; import java.io.Serializable; import java.util.Arrays; -import java.util.Collection; import java.util.List; -import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.CompilationOpContext; @@ -37,6 +33,8 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF; import org.apache.hadoop.hive.ql.udf.generic.UDTFCollector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * UDTFOperator. @@ -137,7 +135,7 @@ public class UDTFOperator extends Operator<UDTFDesc> implements Serializable { @Override public String getName() { - return getOperatorName(); + return UDTFOperator.getOperatorName(); } static public String getOperatorName() { http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java index 3a673e6..39b2776 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java @@ -156,7 +156,7 @@ public class UnionOperator extends Operator<UnionDesc> implements Serializable { */ @Override public String getName() { - return getOperatorName(); + return UnionOperator.getOperatorName(); } static public String getOperatorName() { http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAppMasterEventOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAppMasterEventOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAppMasterEventOperator.java index c591288..1951569 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAppMasterEventOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAppMasterEventOperator.java @@ -18,22 +18,18 @@ package org.apache.hadoop.hive.ql.exec.vector; -import java.util.Collection; -import java.util.concurrent.Future; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.AppMasterEventOperator; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.AppMasterEventDesc; import org.apache.hadoop.hive.ql.plan.OperatorDesc; -import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -import org.apache.hadoop.io.ObjectWritable; -import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.io.Writable; +import com.google.common.annotations.VisibleForTesting; + /** * App Master Event operator implementation. **/ @@ -61,7 +57,8 @@ public class VectorAppMasterEventOperator extends AppMasterEventOperator { } /** Kryo ctor. */ - protected VectorAppMasterEventOperator() { + @VisibleForTesting + public VectorAppMasterEventOperator() { super(); } http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFileSinkOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFileSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFileSinkOperator.java index f09534c..a3082c3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFileSinkOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFileSinkOperator.java @@ -18,9 +18,6 @@ package org.apache.hadoop.hive.ql.exec.vector; -import java.util.Collection; -import java.util.concurrent.Future; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.FileSinkOperator; @@ -29,6 +26,8 @@ import org.apache.hadoop.hive.ql.plan.FileSinkDesc; import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import com.google.common.annotations.VisibleForTesting; + /** * File Sink operator implementation. **/ @@ -56,7 +55,8 @@ public class VectorFileSinkOperator extends FileSinkOperator { } /** Kryo ctor. */ - protected VectorFileSinkOperator() { + @VisibleForTesting + public VectorFileSinkOperator() { super(); } http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFilterOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFilterOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFilterOperator.java index 74a0947..261246b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFilterOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFilterOperator.java @@ -18,9 +18,6 @@ package org.apache.hadoop.hive.ql.exec.vector; -import java.util.Collection; -import java.util.concurrent.Future; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.CompilationOpContext; @@ -32,6 +29,8 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.FilterDesc; import org.apache.hadoop.hive.ql.plan.OperatorDesc; +import com.google.common.annotations.VisibleForTesting; + /** * Filter operator implementation. **/ @@ -57,7 +56,8 @@ public class VectorFilterOperator extends FilterOperator { } /** Kryo ctor. */ - protected VectorFilterOperator() { + @VisibleForTesting + public VectorFilterOperator() { super(); } http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java index 31f5c72..f20f614 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java @@ -22,15 +22,11 @@ import java.lang.management.ManagementFactory; import java.lang.management.MemoryMXBean; import java.lang.ref.SoftReference; import java.util.ArrayList; -import java.util.Collection; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.concurrent.Future; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.CompilationOpContext; @@ -51,6 +47,10 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.io.DataOutputBuffer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.annotations.VisibleForTesting; /** * Vectorized GROUP BY operator implementation. Consumes the vectorized input and @@ -771,7 +771,8 @@ public class VectorGroupByOperator extends Operator<GroupByDesc> implements } /** Kryo ctor. */ - protected VectorGroupByOperator() { + @VisibleForTesting + public VectorGroupByOperator() { super(); } @@ -959,10 +960,6 @@ public class VectorGroupByOperator extends Operator<GroupByDesc> implements } } - static public String getOperatorName() { - return "GBY"; - } - public VectorExpression[] getKeyExpressions() { return keyExpressions; } @@ -988,4 +985,14 @@ public class VectorGroupByOperator extends Operator<GroupByDesc> implements public OperatorType getType() { return OperatorType.GROUPBY; } + + @Override + public String getName() { + return getOperatorName(); + } + + static public String getOperatorName() { + return "GBY"; + } + } http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorLimitOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorLimitOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorLimitOperator.java index 154c647..ea00af3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorLimitOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorLimitOperator.java @@ -24,6 +24,8 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.LimitDesc; import org.apache.hadoop.hive.ql.plan.OperatorDesc; +import com.google.common.annotations.VisibleForTesting; + /** * Limit operator implementation Limits the number of rows to be passed on. **/ @@ -32,7 +34,8 @@ public class VectorLimitOperator extends LimitOperator { private static final long serialVersionUID = 1L; /** Kryo ctor. */ - protected VectorLimitOperator() { + @VisibleForTesting + public VectorLimitOperator() { super(); } http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java index 622f777..e8f4471 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java @@ -19,13 +19,9 @@ package org.apache.hadoop.hive.ql.exec.vector; import java.util.ArrayList; -import java.util.Collection; import java.util.List; import java.util.Map; -import java.util.concurrent.Future; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator; @@ -41,6 +37,10 @@ import org.apache.hadoop.hive.ql.plan.MapJoinDesc; import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.annotations.VisibleForTesting; /** * The vectorized version of the MapJoinOperator. @@ -76,7 +76,8 @@ public class VectorMapJoinOperator extends VectorMapJoinBaseOperator { protected transient Object[] singleRow; /** Kryo ctor. */ - protected VectorMapJoinOperator() { + @VisibleForTesting + public VectorMapJoinOperator() { super(); } http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOuterFilteredOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOuterFilteredOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOuterFilteredOperator.java index 509a43f..0fe1188 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOuterFilteredOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOuterFilteredOperator.java @@ -18,15 +18,14 @@ package org.apache.hadoop.hive.ql.exec.vector; -import java.util.Collection; -import java.util.concurrent.Future; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import com.google.common.annotations.VisibleForTesting; + /** * This is the *NON-NATIVE* vector map join operator for just LEFT OUTER JOIN and filtered. * @@ -50,7 +49,8 @@ public class VectorMapJoinOuterFilteredOperator extends VectorMapJoinBaseOperato protected transient Object[] singleRow; /** Kryo ctor. */ - protected VectorMapJoinOuterFilteredOperator() { + @VisibleForTesting + public VectorMapJoinOuterFilteredOperator() { super(); } http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java index 033be38..9f0c24e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java @@ -24,12 +24,15 @@ import org.apache.hadoop.hive.ql.exec.mr.ExecMapperContext; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.io.Writable; +import com.google.common.annotations.VisibleForTesting; + public class VectorMapOperator extends MapOperator { private static final long serialVersionUID = 1L; /** Kryo ctor. */ - protected VectorMapOperator() { + @VisibleForTesting + public VectorMapOperator() { super(); } http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorReduceSinkOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorReduceSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorReduceSinkOperator.java index b79a3d8..74e5130 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorReduceSinkOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorReduceSinkOperator.java @@ -18,9 +18,6 @@ package org.apache.hadoop.hive.ql.exec.vector; -import java.util.Collection; -import java.util.concurrent.Future; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; @@ -29,6 +26,8 @@ import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import com.google.common.annotations.VisibleForTesting; + public class VectorReduceSinkOperator extends ReduceSinkOperator { private static final long serialVersionUID = 1L; @@ -54,7 +53,8 @@ public class VectorReduceSinkOperator extends ReduceSinkOperator { } /** Kryo ctor. */ - protected VectorReduceSinkOperator() { + @VisibleForTesting + public VectorReduceSinkOperator() { super(); } http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSMBMapJoinOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSMBMapJoinOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSMBMapJoinOperator.java index 9a263e6..85c8506 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSMBMapJoinOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSMBMapJoinOperator.java @@ -19,14 +19,10 @@ package org.apache.hadoop.hive.ql.exec.vector; import java.util.ArrayList; -import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.Future; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator; @@ -40,8 +36,12 @@ import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.SMBJoinDesc; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; -import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.annotations.VisibleForTesting; /** * VectorSMBJoinOperator. @@ -91,7 +91,8 @@ public class VectorSMBMapJoinOperator extends SMBMapJoinOperator implements Vect } /** Kryo ctor. */ - protected VectorSMBMapJoinOperator() { + @VisibleForTesting + public VectorSMBMapJoinOperator() { super(); } http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java index 8db6eba..f7fec8f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java @@ -19,9 +19,7 @@ package org.apache.hadoop.hive.ql.exec.vector; import java.util.ArrayList; -import java.util.Collection; import java.util.List; -import java.util.concurrent.Future; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.CompilationOpContext; @@ -37,6 +35,8 @@ import org.apache.hadoop.hive.ql.plan.api.OperatorType; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import com.google.common.annotations.VisibleForTesting; + /** * Select operator implementation. */ @@ -82,7 +82,8 @@ public class VectorSelectOperator extends Operator<SelectDesc> implements } /** Kryo ctor. */ - protected VectorSelectOperator() { + @VisibleForTesting + public VectorSelectOperator() { super(); } @@ -147,10 +148,6 @@ public class VectorSelectOperator extends Operator<SelectDesc> implements vrg.projectedColumns = originalProjections; } - static public String getOperatorName() { - return "SEL"; - } - public VectorExpression[] getvExpressions() { return vExpressions; } @@ -176,4 +173,14 @@ public class VectorSelectOperator extends Operator<SelectDesc> implements public OperatorType getType() { return OperatorType.SELECT; } + + @Override + public String getName() { + return getOperatorName(); + } + + static public String getOperatorName() { + return "SEL"; + } + } http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSparkHashTableSinkOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSparkHashTableSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSparkHashTableSinkOperator.java index 1e550e7..3d0b571 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSparkHashTableSinkOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSparkHashTableSinkOperator.java @@ -26,8 +26,7 @@ import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.SparkHashTableSinkDesc; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -import java.util.Collection; -import java.util.concurrent.Future; +import com.google.common.annotations.VisibleForTesting; /** * Vectorized version of SparkHashTableSinkOperator @@ -52,7 +51,8 @@ public class VectorSparkHashTableSinkOperator extends SparkHashTableSinkOperator protected transient Object[] singleRow; /** Kryo ctor. */ - protected VectorSparkHashTableSinkOperator() { + @VisibleForTesting + public VectorSparkHashTableSinkOperator() { super(); } http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSparkPartitionPruningSinkOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSparkPartitionPruningSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSparkPartitionPruningSinkOperator.java index 2f02250..e7ac531 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSparkPartitionPruningSinkOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSparkPartitionPruningSinkOperator.java @@ -18,9 +18,6 @@ package org.apache.hadoop.hive.ql.exec.vector; -import java.util.Collection; -import java.util.concurrent.Future; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -31,6 +28,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.io.Writable; +import com.google.common.annotations.VisibleForTesting; + /** * Vectorized version for SparkPartitionPruningSinkOperator. * Forked from VectorAppMasterEventOperator. @@ -55,7 +54,8 @@ public class VectorSparkPartitionPruningSinkOperator extends SparkPartitionPruni } /** Kryo ctor. */ - protected VectorSparkPartitionPruningSinkOperator() { + @VisibleForTesting + public VectorSparkPartitionPruningSinkOperator() { super(); } http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java index 737d9c3..d3ec7ff 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java @@ -35,8 +35,6 @@ import java.util.List; import java.util.Map; import java.util.Stack; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.ql.exec.FilterOperator; @@ -64,11 +62,15 @@ import org.apache.hadoop.hive.ql.plan.AggregationDesc; import org.apache.hadoop.hive.ql.plan.BaseWork; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; +import org.apache.hadoop.hive.ql.plan.GroupByDesc; import org.apache.hadoop.hive.ql.plan.MapWork; import org.apache.hadoop.hive.ql.plan.PartitionDesc; import org.apache.hadoop.hive.ql.plan.ReduceWork; +import org.apache.hadoop.hive.ql.plan.SelectDesc; import org.apache.hadoop.hive.ql.plan.Statistics; import org.apache.hadoop.hive.ql.plan.TezWork; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * LlapDecider takes care of tagging certain vertices in the execution @@ -319,7 +321,8 @@ public class LlapDecider implements PhysicalPlanResolver { @Override public Object process(Node n, Stack<Node> s, NodeProcessorCtx c, Object... os) { - List<AggregationDesc> aggs = ((GroupByOperator)n).getConf().getAggregators(); + @SuppressWarnings("unchecked") + List<AggregationDesc> aggs = ((Operator<GroupByDesc>) n).getConf().getAggregators(); return new Boolean(checkAggregators(aggs)); } }); @@ -328,7 +331,8 @@ public class LlapDecider implements PhysicalPlanResolver { @Override public Object process(Node n, Stack<Node> s, NodeProcessorCtx c, Object... os) { - List<ExprNodeDesc> exprs = ((SelectOperator)n).getConf().getColList(); + @SuppressWarnings({ "unchecked" }) + List<ExprNodeDesc> exprs = ((Operator<SelectDesc>) n).getConf().getColList(); return new Boolean(checkExpressions(exprs)); } }); http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkPartitionPruningSinkOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkPartitionPruningSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkPartitionPruningSinkOperator.java index 3f31fb5..dd8ff01 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkPartitionPruningSinkOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkPartitionPruningSinkOperator.java @@ -21,11 +21,7 @@ package org.apache.hadoop.hive.ql.parse.spark; import java.io.BufferedOutputStream; import java.io.IOException; import java.io.ObjectOutputStream; -import java.util.Collection; -import java.util.concurrent.Future; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; @@ -36,11 +32,15 @@ import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.optimizer.spark.SparkPartitionPruningSinkDesc; import org.apache.hadoop.hive.ql.plan.api.OperatorType; +import org.apache.hadoop.hive.serde2.Serializer; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.Writable; import org.apache.hadoop.util.ReflectionUtils; -import org.apache.hadoop.hive.serde2.Serializer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.annotations.VisibleForTesting; /** * This operator gets partition info from the upstream operators, and write them @@ -55,7 +55,8 @@ public class SparkPartitionPruningSinkOperator extends Operator<SparkPartitionPr protected static final Logger LOG = LoggerFactory.getLogger(SparkPartitionPruningSinkOperator.class); /** Kryo ctor. */ - protected SparkPartitionPruningSinkOperator() { + @VisibleForTesting + public SparkPartitionPruningSinkOperator() { super(); } @@ -63,6 +64,7 @@ public class SparkPartitionPruningSinkOperator extends Operator<SparkPartitionPr super(ctx); } + @Override @SuppressWarnings("deprecation") public void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); @@ -141,7 +143,7 @@ public class SparkPartitionPruningSinkOperator extends Operator<SparkPartitionPr @Override public String getName() { - return getOperatorName(); + return SparkPartitionPruningSinkOperator.getOperatorName(); } public static String getOperatorName() { http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperatorNames.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperatorNames.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperatorNames.java new file mode 100644 index 0000000..e936380 --- /dev/null +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperatorNames.java @@ -0,0 +1,98 @@ +package org.apache.hadoop.hive.ql.exec; + +import junit.framework.TestCase; + +import org.apache.hadoop.hive.ql.exec.vector.VectorAppMasterEventOperator; +import org.apache.hadoop.hive.ql.exec.vector.VectorFilterOperator; +import org.apache.hadoop.hive.ql.exec.vector.VectorGroupByOperator; +import org.apache.hadoop.hive.ql.exec.vector.VectorLimitOperator; +import org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOperator; +import org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOuterFilteredOperator; +import org.apache.hadoop.hive.ql.exec.vector.VectorMapOperator; +import org.apache.hadoop.hive.ql.exec.vector.VectorSMBMapJoinOperator; +import org.apache.hadoop.hive.ql.exec.vector.VectorSelectOperator; +import org.apache.hadoop.hive.ql.exec.vector.VectorSparkHashTableSinkOperator; +import org.apache.hadoop.hive.ql.exec.vector.VectorSparkPartitionPruningSinkOperator; +import org.apache.hadoop.hive.ql.parse.spark.SparkPartitionPruningSinkOperator; +import org.junit.Test; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +public class TestOperatorNames extends TestCase { + + public TestOperatorNames(String name) { + super(name); + } + + @Override + protected void setUp() throws Exception { + super.setUp(); + } + + @Override + protected void tearDown() throws Exception { + super.tearDown(); + } + + /* + * If there's a mismatch between static and object name, or a mismatch between + * vector and non-vector operator name, the optimizer doens't work correctly. + */ + @Test + public void testOperatorNames() throws Exception { + + assertEquals(SelectOperator.getOperatorName(), new SelectOperator().getName()); + assertEquals(SelectOperator.getOperatorName(), new VectorSelectOperator().getName()); + + assertEquals(GroupByOperator.getOperatorName(), new GroupByOperator().getName()); + assertEquals(GroupByOperator.getOperatorName(), new VectorGroupByOperator().getName()); + + assertEquals(FilterOperator.getOperatorName(), new FilterOperator().getName()); + assertEquals(FilterOperator.getOperatorName(), new VectorFilterOperator().getName()); + + assertEquals(LimitOperator.getOperatorName(), new LimitOperator().getName()); + assertEquals(LimitOperator.getOperatorName(), new VectorLimitOperator().getName()); + + assertEquals(MapOperator.getOperatorName(), new MapOperator().getName()); + assertEquals(MapOperator.getOperatorName(), new VectorMapOperator().getName()); + + assertEquals(MapJoinOperator.getOperatorName(), new MapJoinOperator().getName()); + assertEquals(MapJoinOperator.getOperatorName(), new VectorMapJoinOperator().getName()); + + assertEquals(AppMasterEventOperator.getOperatorName(), new AppMasterEventOperator().getName()); + assertEquals(AppMasterEventOperator.getOperatorName(), + new VectorAppMasterEventOperator().getName()); + + assertEquals(SMBMapJoinOperator.getOperatorName(), new SMBMapJoinOperator().getName()); + assertEquals(SMBMapJoinOperator.getOperatorName(), new VectorSMBMapJoinOperator().getName()); + + assertEquals(MapJoinOperator.getOperatorName(), + new VectorMapJoinOuterFilteredOperator().getName()); + + assertEquals(SparkHashTableSinkOperator.getOperatorName(), + new SparkHashTableSinkOperator().getName()); + assertEquals(SparkHashTableSinkOperator.getOperatorName(), + new VectorSparkHashTableSinkOperator().getName()); + + assertEquals(SparkPartitionPruningSinkOperator.getOperatorName(), + new SparkPartitionPruningSinkOperator().getName()); + assertEquals(SparkPartitionPruningSinkOperator.getOperatorName(), + new VectorSparkPartitionPruningSinkOperator().getName()); + + } + +} http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeCaptureOutputOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeCaptureOutputOperator.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeCaptureOutputOperator.java index 74e077b..bdf911c 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeCaptureOutputOperator.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeCaptureOutputOperator.java @@ -20,9 +20,7 @@ package org.apache.hadoop.hive.ql.exec.vector.util; import java.io.Serializable; import java.util.ArrayList; -import java.util.Collection; import java.util.List; -import java.util.concurrent.Future; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.CompilationOpContext; @@ -102,4 +100,12 @@ public class FakeCaptureOutputOperator extends Operator<FakeCaptureOutputDesc> return null; } + @Override + public String getName() { + return FakeCaptureOutputOperator.getOperatorName(); + } + + public static String getOperatorName() { + return "FAKE_CAPTURE"; + } } http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeVectorDataSourceOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeVectorDataSourceOperator.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeVectorDataSourceOperator.java index d06d214..a2032bf 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeVectorDataSourceOperator.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeVectorDataSourceOperator.java @@ -20,9 +20,7 @@ package org.apache.hadoop.hive.ql.exec.vector.util; import java.io.Serializable; import java.util.ArrayList; -import java.util.Collection; import java.util.List; -import java.util.concurrent.Future; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.CompilationOpContext; @@ -89,4 +87,13 @@ public class FakeVectorDataSourceOperator extends Operator<FakeVectorDataSourceO public OperatorType getType() { return null; } + + @Override + public String getName() { + return FakeVectorDataSourceOperator.getOperatorName(); + } + + public static String getOperatorName() { + return "FAKE_VECTOR_DS"; + } } http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/test/results/clientpositive/explain_logical.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/explain_logical.q.out b/ql/src/test/results/clientpositive/explain_logical.q.out index bf35cd5..dc78480 100644 --- a/ql/src/test/results/clientpositive/explain_logical.q.out +++ b/ql/src/test/results/clientpositive/explain_logical.q.out @@ -449,7 +449,7 @@ src expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - ListSink (OP_3) + ListSink (LIST_SINK_3) PREHOOK: query: EXPLAIN LOGICAL SELECT * FROM V2 PREHOOK: type: QUERY @@ -480,7 +480,7 @@ srcpart expressions: ds (type: string), key (type: string), value (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - ListSink (OP_5) + ListSink (LIST_SINK_5) PREHOOK: query: EXPLAIN LOGICAL SELECT * FROM V3 PREHOOK: type: QUERY @@ -723,7 +723,7 @@ srcpart expressions: key (type: string), value (type: string), '10' (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - ListSink (OP_5) + ListSink (LIST_SINK_5) PREHOOK: query: EXPLAIN LOGICAL SELECT s1.key, s1.cnt, s2.value FROM (SELECT key, count(value) as cnt FROM src GROUP BY key) s1 JOIN src s2 ON (s1.key = s2.key) ORDER BY s1.key PREHOOK: type: QUERY http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/test/results/clientpositive/tez/explainuser_3.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/tez/explainuser_3.q.out b/ql/src/test/results/clientpositive/tez/explainuser_3.q.out index 1222b94..f4e21bd 100644 --- a/ql/src/test/results/clientpositive/tez/explainuser_3.q.out +++ b/ql/src/test/results/clientpositive/tez/explainuser_3.q.out @@ -31,11 +31,11 @@ Stage-0 Stage-1 Reducer 2 vectorized File Output Operator [FS_8] - Select Operator [OP_7] (rows=10 width=170) + Select Operator [SEL_7] (rows=10 width=170) Output:["_col0","_col1"] <-Map 1 [SIMPLE_EDGE] vectorized SHUFFLE [RS_6] - Select Operator [OP_5] (rows=10 width=170) + Select Operator [SEL_5] (rows=10 width=170) Output:["_col0","_col1"] TableScan [TS_0] (rows=10 width=170) default@acid_vectorized,acid_vectorized, ACID table,Tbl:COMPLETE,Col:NONE,Output:["a","b"] @@ -457,7 +457,7 @@ Stage-0 File Output Operator [FS_8] Limit [LIM_7] (rows=5 width=10) Number of rows:5 - Select Operator [OP_6] (rows=500 width=10) + Select Operator [SEL_6] (rows=500 width=10) Output:["_col0","_col1"] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_2] @@ -503,7 +503,7 @@ Stage-3 Map 1 vectorized File Output Operator [FS_10] table:{"name:":"default.orc_merge5"} - Select Operator [OP_9] (rows=306 width=268) + Select Operator [SEL_9] (rows=306 width=268) Output:["_col0","_col1","_col2","_col3","_col4"] Filter Operator [FIL_8] (rows=306 width=268) predicate:(userid <= 13) http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/test/results/clientpositive/tez/vector_aggregate_without_gby.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/tez/vector_aggregate_without_gby.q.out b/ql/src/test/results/clientpositive/tez/vector_aggregate_without_gby.q.out index 55a3842..ab627b5 100644 --- a/ql/src/test/results/clientpositive/tez/vector_aggregate_without_gby.q.out +++ b/ql/src/test/results/clientpositive/tez/vector_aggregate_without_gby.q.out @@ -48,13 +48,13 @@ Stage-0 Stage-1 Reducer 2 vectorized File Output Operator [FS_7] - Group By Operator [OP_12] (rows=1 width=88) + Group By Operator [GBY_12] (rows=1 width=88) Output:["_col0","_col1"],aggregations:["max(VALUE._col0)","max(VALUE._col1)"] <-Map 1 [SIMPLE_EDGE] vectorized SHUFFLE [RS_4] - Group By Operator [OP_11] (rows=1 width=88) + Group By Operator [GBY_11] (rows=1 width=88) Output:["_col0","_col1"],aggregations:["max(dt)","max(greg_dt)"] - Select Operator [OP_10] (rows=3 width=102) + Select Operator [SEL_10] (rows=3 width=102) Output:["dt","greg_dt"] Filter Operator [FIL_9] (rows=3 width=102) predicate:(id = 5) http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/test/results/clientpositive/tez/vector_auto_smb_mapjoin_14.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/tez/vector_auto_smb_mapjoin_14.q.out b/ql/src/test/results/clientpositive/tez/vector_auto_smb_mapjoin_14.q.out index 7175be0..67ddd9e 100644 --- a/ql/src/test/results/clientpositive/tez/vector_auto_smb_mapjoin_14.q.out +++ b/ql/src/test/results/clientpositive/tez/vector_auto_smb_mapjoin_14.q.out @@ -65,7 +65,7 @@ Stage-0 Stage-1 Reducer 2 vectorized File Output Operator [FS_14] - Group By Operator [OP_21] (rows=1 width=8) + Group By Operator [GBY_21] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_11] @@ -135,14 +135,14 @@ Stage-0 Stage-1 Reducer 3 vectorized File Output Operator [FS_19] - Group By Operator [OP_29] (rows=1 width=8) + Group By Operator [GBY_29] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] <-Reducer 2 [SIMPLE_EDGE] vectorized SHUFFLE [RS_16] - Group By Operator [OP_28] (rows=1 width=8) + Group By Operator [GBY_28] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] - Select Operator [OP_27] (rows=5 width=93) - Group By Operator [OP_26] (rows=5 width=93) + Select Operator [SEL_27] (rows=5 width=93) + Group By Operator [GBY_26] (rows=5 width=93) Output:["_col0"],keys:KEY._col0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_11] @@ -247,7 +247,7 @@ Stage-0 <-Reducer 2 [SIMPLE_EDGE] vectorized SHUFFLE [RS_51] PartitionCols:_col0 - Group By Operator [OP_50] (rows=5 width=93) + Group By Operator [GBY_50] (rows=5 width=93) Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_11] @@ -271,7 +271,7 @@ Stage-0 <-Reducer 6 [SIMPLE_EDGE] vectorized SHUFFLE [RS_53] PartitionCols:_col0 - Group By Operator [OP_52] (rows=5 width=93) + Group By Operator [GBY_52] (rows=5 width=93) Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 <-Map 5 [SIMPLE_EDGE] SHUFFLE [RS_25] @@ -366,7 +366,7 @@ Stage-0 Stage-1 Reducer 2 vectorized File Output Operator [FS_14] - Group By Operator [OP_21] (rows=1 width=8) + Group By Operator [GBY_21] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_11] @@ -445,7 +445,7 @@ Stage-0 Stage-1 Reducer 2 vectorized File Output Operator [FS_14] - Group By Operator [OP_21] (rows=1 width=8) + Group By Operator [GBY_21] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_11] @@ -548,7 +548,7 @@ Stage-0 Stage-1 Reducer 2 vectorized File Output Operator [FS_14] - Group By Operator [OP_21] (rows=1 width=8) + Group By Operator [GBY_21] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_11] @@ -641,7 +641,7 @@ Stage-0 Stage-1 Reducer 2 vectorized File Output Operator [FS_14] - Group By Operator [OP_21] (rows=1 width=8) + Group By Operator [GBY_21] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_11] @@ -711,7 +711,7 @@ Stage-0 Stage-1 Reducer 3 vectorized File Output Operator [FS_14] - Group By Operator [OP_28] (rows=1 width=8) + Group By Operator [GBY_28] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_11] @@ -722,7 +722,7 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] vectorized SHUFFLE [RS_24] PartitionCols:_col0 - Select Operator [OP_23] (rows=10 width=93) + Select Operator [SEL_23] (rows=10 width=93) Output:["_col0"] Filter Operator [FIL_22] (rows=10 width=93) predicate:(key + 1) is not null @@ -731,7 +731,7 @@ Stage-0 <-Map 4 [SIMPLE_EDGE] vectorized SHUFFLE [RS_27] PartitionCols:_col0 - Select Operator [OP_26] (rows=10 width=93) + Select Operator [SEL_26] (rows=10 width=93) Output:["_col0"] Filter Operator [FIL_25] (rows=10 width=93) predicate:(key + 1) is not null @@ -782,7 +782,7 @@ Stage-0 Stage-1 Reducer 2 vectorized File Output Operator [FS_14] - Group By Operator [OP_21] (rows=1 width=8) + Group By Operator [GBY_21] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_11] @@ -853,7 +853,7 @@ Stage-0 Stage-1 Reducer 2 vectorized File Output Operator [FS_18] - Group By Operator [OP_31] (rows=1 width=8) + Group By Operator [GBY_31] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_15] @@ -946,7 +946,7 @@ Stage-0 Stage-1 Reducer 2 vectorized File Output Operator [FS_14] - Group By Operator [OP_21] (rows=1 width=8) + Group By Operator [GBY_21] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_11] @@ -1204,9 +1204,9 @@ Stage-4 Reducer 2 vectorized File Output Operator [FS_25] table:{"name:":"default.dest2"} - Select Operator [OP_24] (rows=5 width=93) + Select Operator [SEL_24] (rows=5 width=93) Output:["_col0","_col1"] - Group By Operator [OP_23] (rows=5 width=93) + Group By Operator [GBY_23] (rows=5 width=93) Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 <-Map 1 [SIMPLE_EDGE] File Output Operator [FS_9] http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/test/results/clientpositive/tez/vector_groupby_mapjoin.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/tez/vector_groupby_mapjoin.q.out b/ql/src/test/results/clientpositive/tez/vector_groupby_mapjoin.q.out index 5c3e198..d406f2b 100644 --- a/ql/src/test/results/clientpositive/tez/vector_groupby_mapjoin.q.out +++ b/ql/src/test/results/clientpositive/tez/vector_groupby_mapjoin.q.out @@ -28,7 +28,7 @@ Stage-0 Stage-1 Reducer 2 vectorized File Output Operator [FS_34] - Select Operator [OP_33] (rows=302 width=10) + Select Operator [SEL_33] (rows=302 width=10) Output:["_col0","_col1"] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_22] @@ -52,7 +52,7 @@ Stage-0 Select Operator [SEL_10] (rows=1 width=8) Filter Operator [FIL_9] (rows=1 width=8) predicate:(_col0 = 0) - Group By Operator [OP_32] (rows=1 width=8) + Group By Operator [GBY_32] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] <-Map 3 [SIMPLE_EDGE] SHUFFLE [RS_6] http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/test/results/clientpositive/tez/vector_join_part_col_char.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/tez/vector_join_part_col_char.q.out b/ql/src/test/results/clientpositive/tez/vector_join_part_col_char.q.out index 7604538..93137f1 100644 --- a/ql/src/test/results/clientpositive/tez/vector_join_part_col_char.q.out +++ b/ql/src/test/results/clientpositive/tez/vector_join_part_col_char.q.out @@ -117,20 +117,20 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] vectorized SHUFFLE [RS_23] PartitionCols:_col2 - Select Operator [OP_22] (rows=2 width=102) + Select Operator [SEL_22] (rows=2 width=102) Output:["_col0","_col1","_col2"] TableScan [TS_0] (rows=2 width=102) default@char_tbl1,c1,Tbl:COMPLETE,Col:NONE,Output:["name","age"] Dynamic Partitioning Event Operator [EVENT_20] (rows=2 width=102) - Group By Operator [OP_25] (rows=2 width=102) + Group By Operator [GBY_25] (rows=2 width=102) Output:["_col0"],keys:_col0 - Select Operator [OP_24] (rows=2 width=102) + Select Operator [SEL_24] (rows=2 width=102) Output:["_col0"] - Please refer to the previous Select Operator [OP_22] + Please refer to the previous Select Operator [SEL_22] <-Map 3 [SIMPLE_EDGE] vectorized SHUFFLE [RS_27] PartitionCols:_col2 - Select Operator [OP_26] (rows=2 width=101) + Select Operator [SEL_26] (rows=2 width=101) Output:["_col0","_col1","_col2"] TableScan [TS_3] (rows=2 width=101) default@char_tbl2,c2,Tbl:COMPLETE,Col:NONE,Output:["name","age"] http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/test/results/clientpositive/tez/vectorized_parquet_types.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/tez/vectorized_parquet_types.q.out b/ql/src/test/results/clientpositive/tez/vectorized_parquet_types.q.out index 8355381..56a01b7 100644 --- a/ql/src/test/results/clientpositive/tez/vectorized_parquet_types.q.out +++ b/ql/src/test/results/clientpositive/tez/vectorized_parquet_types.q.out @@ -251,7 +251,7 @@ Stage-0 Stage-1 Reducer 3 vectorized File Output Operator [FS_10] - Select Operator [OP_9] (rows=11 width=11) + Select Operator [SEL_9] (rows=11 width=11) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_6] http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/test/results/clientpositive/tez/vectorized_timestamp.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/tez/vectorized_timestamp.q.out b/ql/src/test/results/clientpositive/tez/vectorized_timestamp.q.out index 5382865..fb272dd 100644 --- a/ql/src/test/results/clientpositive/tez/vectorized_timestamp.q.out +++ b/ql/src/test/results/clientpositive/tez/vectorized_timestamp.q.out @@ -101,7 +101,7 @@ Stage-0 Stage-1 Map 1 vectorized File Output Operator [FS_4] - Select Operator [OP_3] (rows=2 width=40) + Select Operator [SEL_3] (rows=2 width=40) Output:["_col0"] TableScan [TS_0] (rows=2 width=40) default@test,test,Tbl:COMPLETE,Col:NONE,Output:["ts"] @@ -135,13 +135,13 @@ Stage-0 File Output Operator [FS_6] Select Operator [SEL_5] (rows=1 width=80) Output:["_col0","_col1","_col2"] - Group By Operator [OP_9] (rows=1 width=80) + Group By Operator [GBY_9] (rows=1 width=80) Output:["_col0","_col1"],aggregations:["min(VALUE._col0)","max(VALUE._col1)"] <-Map 1 [SIMPLE_EDGE] vectorized SHUFFLE [RS_3] - Group By Operator [OP_8] (rows=1 width=80) + Group By Operator [GBY_8] (rows=1 width=80) Output:["_col0","_col1"],aggregations:["min(ts)","max(ts)"] - Select Operator [OP_7] (rows=2 width=40) + Select Operator [SEL_7] (rows=2 width=40) Output:["ts"] TableScan [TS_0] (rows=2 width=40) default@test,test,Tbl:COMPLETE,Col:NONE,Output:["ts"]