This is an automated email from the ASF dual-hosted git repository.
gopalv pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 8215cf6 HIVE-24666: Vectorized UDFToBoolean may unable to filter rows
if input is string (Zhihua Deng, reviewed by Gopal V)
8215cf6 is described below
commit 8215cf6b0ccf96234837292d46d71030c90090a3
Author: dengzhhu653 <dengzhhu...@gmail.com>
AuthorDate: Tue Feb 16 13:28:21 2021 -0800
HIVE-24666: Vectorized UDFToBoolean may unable to filter rows if input is
string (Zhihua Deng, reviewed by Gopal V)
Signed-off-by: Gopal V <gop...@apache.org>
---
.../hive/ql/exec/vector/VectorizationContext.java | 64 ++--
.../exec/vector/expressions/FuncStringToLong.java | 4 +-
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 15 +
.../hive/ql/parse/type/TypeCheckProcFactory.java | 23 ++
.../expressions/TestVectorFilterExpressions.java | 55 +++
.../queries/clientpositive/vector_cast_filter.q | 34 ++
.../llap/annotate_stats_filter.q.out | 10 +-
..._empty_where.q.out => vector_cast_filter.q.out} | 426 ++++++++++++++-------
.../clientpositive/llap/vector_empty_where.q.out | 20 +-
9 files changed, 462 insertions(+), 189 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
index 9958f66..4af7325 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
@@ -871,28 +871,11 @@ import com.google.common.annotations.VisibleForTesting;
} else {
// Ok, we need to convert.
List<ExprNodeDesc> exprAsList = Collections.singletonList(exprDesc);
-
- // First try our cast method that will handle a few special cases.
- VectorExpression castToBooleanExpr = getCastToBoolean(exprAsList);
- if (castToBooleanExpr == null) {
-
- // Ok, try the UDF.
- castToBooleanExpr = getVectorExpressionForUdf(null,
UDFToBoolean.class, exprAsList,
- VectorExpressionDescriptor.Mode.PROJECTION,
TypeInfoFactory.booleanTypeInfo);
- if (castToBooleanExpr == null) {
- throw new HiveException("Cannot vectorize converting expression " +
- exprDesc.getExprString() + " to boolean");
- }
+ expr = getCastToBooleanExpression(exprAsList,
VectorExpressionDescriptor.Mode.FILTER);
+ if (expr == null) {
+ throw new HiveException("Cannot vectorize converting expression " +
+ exprDesc.getExprString() + " to boolean");
}
-
- final int outputColumnNum = castToBooleanExpr.getOutputColumnNum();
-
- expr = new SelectColumnIsTrue(outputColumnNum);
-
- expr.setChildExpressions(new VectorExpression[] {castToBooleanExpr});
-
- expr.setInputTypeInfos(castToBooleanExpr.getOutputTypeInfo());
- expr.setInputDataTypePhysicalVariations(DataTypePhysicalVariation.NONE);
}
return expr;
}
@@ -1483,6 +1466,10 @@ import com.google.common.annotations.VisibleForTesting;
// Boolean is purposely excluded.
}
+ public static boolean isCastToBoolean(Class<? extends UDF> udfClass) {
+ return udfClass.equals(UDFToBoolean.class);
+ }
+
public static boolean isCastToFloatFamily(Class<? extends UDF> udfClass) {
return udfClass.equals(UDFToDouble.class)
|| udfClass.equals(UDFToFloat.class);
@@ -1598,7 +1585,11 @@ import com.google.common.annotations.VisibleForTesting;
if (typeInfo.getCategory() != Category.PRIMITIVE) {
throw new HiveException("Complex type constants (" +
typeInfo.getCategory() + ") not supported for type name " + typeName);
}
- return new ConstantVectorExpression(outCol, typeInfo, true);
+ if (mode == VectorExpressionDescriptor.Mode.FILTER) {
+ return new FilterConstantBooleanVectorExpression(0);
+ } else {
+ return new ConstantVectorExpression(outCol, typeInfo, true);
+ }
}
// Boolean is special case.
@@ -2992,8 +2983,8 @@ import com.google.common.annotations.VisibleForTesting;
PrimitiveCategory integerPrimitiveCategory =
getAnyIntegerPrimitiveCategoryFromUdfClass(cl);
ve = getCastToLongExpression(childExpr, integerPrimitiveCategory);
- } else if (cl.equals(UDFToBoolean.class)) {
- ve = getCastToBoolean(childExpr);
+ } else if (isCastToBoolean(cl)) {
+ ve = getCastToBooleanExpression(childExpr, mode);
} else if (isCastToFloatFamily(cl)) {
ve = getCastToDoubleExpression(cl, childExpr, returnType);
}
@@ -3470,26 +3461,41 @@ import com.google.common.annotations.VisibleForTesting;
return null;
}
- private VectorExpression getCastToBoolean(List<ExprNodeDesc> childExpr)
+ private VectorExpression getCastToBooleanExpression(List<ExprNodeDesc>
childExpr, VectorExpressionDescriptor.Mode mode)
throws HiveException {
ExprNodeDesc child = childExpr.get(0);
TypeInfo inputTypeInfo = child.getTypeInfo();
String inputType = inputTypeInfo.toString();
if (child instanceof ExprNodeConstantDesc) {
if (null == ((ExprNodeConstantDesc)child).getValue()) {
- return getConstantVectorExpression(null,
TypeInfoFactory.booleanTypeInfo, VectorExpressionDescriptor.Mode.PROJECTION);
+ return getConstantVectorExpression(null,
TypeInfoFactory.booleanTypeInfo, mode);
}
// Don't do constant folding here. Wait until the optimizer is changed
to do it.
// Family of related JIRAs: HIVE-7421, HIVE-7422, and HIVE-7424.
return null;
}
+
+ VectorExpression ve;
// Long and double are handled using descriptors, string needs to be
specially handled.
if (isStringFamily(inputType)) {
-
- return createVectorExpression(CastStringToBoolean.class, childExpr,
+ ve = createVectorExpression(CastStringToBoolean.class, childExpr,
VectorExpressionDescriptor.Mode.PROJECTION,
TypeInfoFactory.booleanTypeInfo, DataTypePhysicalVariation.NONE);
+ } else {
+ // Ok, try the UDF.
+ ve = getVectorExpressionForUdf(null, UDFToBoolean.class, childExpr,
+ VectorExpressionDescriptor.Mode.PROJECTION,
TypeInfoFactory.booleanTypeInfo);
}
- return null;
+
+ if (ve == null || mode == VectorExpressionDescriptor.Mode.PROJECTION) {
+ return ve;
+ }
+
+ int outputColumnNum = ve.getOutputColumnNum();
+ SelectColumnIsTrue filterVectorExpr = new
SelectColumnIsTrue(outputColumnNum);
+ filterVectorExpr.setChildExpressions(new VectorExpression[] { ve });
+ filterVectorExpr.setInputTypeInfos(ve.getOutputTypeInfo());
+
filterVectorExpr.setInputDataTypePhysicalVariations(DataTypePhysicalVariation.NONE);
+ return filterVectorExpr;
}
private VectorExpression getCastToLongExpression(List<ExprNodeDesc>
childExpr, PrimitiveCategory integerPrimitiveCategory)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FuncStringToLong.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FuncStringToLong.java
index fffb36d..007cbd2 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FuncStringToLong.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FuncStringToLong.java
@@ -27,8 +27,8 @@ import
org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.metadata.HiveException;
/**
- * Superclass to support vectorized functions that take a long
- * and return a string, optionally with additional configuration arguments.
+ * Superclass to support vectorized functions that take a string
+ * and return a long, optionally with additional configuration arguments.
* Used for cast(string), length(string), etc
*/
public abstract class FuncStringToLong extends VectorExpression {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index d15e46a..7c73065 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -3583,6 +3583,21 @@ public class SemanticAnalyzer extends
BaseSemanticAnalyzer {
}
}
+ if (!filterCond.getTypeInfo().accept(TypeInfoFactory.booleanTypeInfo)) {
+ // If the returning type of the filter condition is not boolean, try to
implicitly
+ // convert the result of the condition to a boolean value.
+ if (filterCond.getTypeInfo().getCategory() ==
ObjectInspector.Category.PRIMITIVE) {
+ // For primitive types like string/double/timestamp, try to cast the
result of
+ // the child expression to a boolean.
+ filterCond = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor()
+ .createConversionCast(filterCond, TypeInfoFactory.booleanTypeInfo);
+ } else {
+ // For complex types like map/list/struct, create a isnotnull function
on the child expression.
+ filterCond = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor()
+ .getFuncExprNodeDesc("isnotnull", filterCond);
+ }
+ }
+
Operator output = putOpInsertMap(OperatorFactory.getAndMakeChild(
new FilterDesc(filterCond, false), new RowSchema(
inputRR.getColumnInfos()), input), inputRR);
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/parse/type/TypeCheckProcFactory.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/type/TypeCheckProcFactory.java
index aa130c8..488dc50 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/type/TypeCheckProcFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/type/TypeCheckProcFactory.java
@@ -838,6 +838,29 @@ public class TypeCheckProcFactory<T> {
children.set(0, newColumn);
}
}
+
+ if (funcText.equalsIgnoreCase("and") || funcText.equalsIgnoreCase("or")
+ || funcText.equalsIgnoreCase("not") ||
funcText.equalsIgnoreCase("!")) {
+ // If the current function is a conjunction, the returning types of
the children should be booleans.
+ // Iterate on the children, if the result of a child expression is not
a boolean, try to implicitly
+ // convert the result of such a child to a boolean value.
+ for (int i = 0; i < children.size(); i++) {
+ T child = children.get(i);
+ TypeInfo typeInfo = exprFactory.getTypeInfo(child);
+ if (!TypeInfoFactory.booleanTypeInfo.accept(typeInfo)) {
+ if (typeInfo.getCategory() == ObjectInspector.Category.PRIMITIVE) {
+ // For primitive types like string/double/timestamp, try to cast
the result of
+ // the child expression to a boolean.
+ children.set(i, createConversionCast(child,
TypeInfoFactory.booleanTypeInfo));
+ } else {
+ // For complex types like map/list/struct, create a isnotnull
function on the child expression.
+ child =
exprFactory.createFuncCallExpr(TypeInfoFactory.booleanTypeInfo,
+ exprFactory.getFunctionInfo("isnotnull"),"isnotnull",
Arrays.asList(child));
+ children.set(i, child);
+ }
+ }
+ }
+ }
}
protected T getXpathOrFuncExprNodeDesc(ASTNode node,
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterExpressions.java
b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterExpressions.java
index c8c75e0..d9d56a0 100644
---
a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterExpressions.java
+++
b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterExpressions.java
@@ -33,6 +33,7 @@ import
org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import
org.apache.hadoop.hive.ql.exec.vector.expressions.gen.CastLongToBooleanViaLongToLong;
import
org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterDecimalColGreaterEqualDecimalColumn;
import
org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterDecimalColLessDecimalScalar;
import
org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterDecimalScalarGreaterDecimalColumn;
@@ -1057,4 +1058,58 @@ public class TestVectorFilterExpressions {
b.size = 3;
return b;
}
+
+ // Test that the cast filter should be wrapped in SelectColumnIsTrue
+ @Test
+ public void testCastFilter() throws HiveException {
+ int seed = 0;
+ VectorizedRowBatch vrb = VectorizedRowGroupGenUtil.getVectorizedRowBatch(
+ 3, 4, seed);
+ vrb.cols[0] = new BytesColumnVector();
+ BytesColumnVector bcv = (BytesColumnVector) vrb.cols[0];
+ bcv.initBuffer();
+ byte[] n = "no".getBytes();
+ byte[] f = "false".getBytes();
+ bcv.setVal(0, n, 0, n.length);
+ bcv.setVal(1, f, 0, f.length);
+ bcv.setVal(2, c, 0, 1);
+
+ VectorExpression ve1 = new CastStringToBoolean(0,2);
+ VectorExpression ve2 = new CastLongToBooleanViaLongToLong(1, 3);
+ VectorExpression orExpr = new FilterExprOrExpr();
+ orExpr.setChildExpressions(new VectorExpression[] {ve1, ve2});
+ orExpr.evaluate(vrb);
+
+ // Only one row should be filtered out, but both filters fail to take
effect
+ assertFalse(vrb.selectedInUse);
+ assertEquals(0, vrb.selected[0]);
+ assertEquals(1, vrb.selected[1]);
+ assertEquals(2, vrb.selected[2]);
+ assertEquals(3, vrb.size);
+
+ SelectColumnIsTrue filter1 = new SelectColumnIsTrue(2);
+ filter1.setChildExpressions(new VectorExpression[]{ ve1 });
+ VectorExpression andExpr = new FilterExprAndExpr();
+ // SelectColumnIsTrue(cast string) and CastLongToBooleanViaLongToLong
+ andExpr.setChildExpressions(new VectorExpression[]{filter1, ve2});
+ andExpr.evaluate(vrb);
+
+ // All should be filtered out, but CastLongToBooleanViaLongToLong fails to
take effect
+ assertTrue(vrb.selectedInUse);
+ assertEquals(2, vrb.selected[0]);
+ assertEquals(1, vrb.size);
+
+ // restore
+ vrb.selectedInUse = false;
+ vrb.size = 3;
+
+ SelectColumnIsTrue filter2 = new SelectColumnIsTrue(3);
+ filter2.setChildExpressions(new VectorExpression[]{ ve2 });
+ andExpr.setChildExpressions(new VectorExpression[]{filter1, filter2});
+ andExpr.evaluate(vrb);
+
+ assertTrue(vrb.selectedInUse);
+ assertEquals(0, vrb.size);
+ }
+
}
diff --git a/ql/src/test/queries/clientpositive/vector_cast_filter.q
b/ql/src/test/queries/clientpositive/vector_cast_filter.q
new file mode 100644
index 0000000..df60419
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/vector_cast_filter.q
@@ -0,0 +1,34 @@
+--! qt:dataset:alltypesorc
+SET hive.vectorized.execution.enabled=false;
+
+explain
+select count (distinct cint) from alltypesorc where !cstring1;
+
+select count (distinct cint) from alltypesorc where !cstring1;
+
+explain
+select count (distinct cint) from alltypesorc where cint and cstring1;
+
+select count (distinct cint) from alltypesorc where cint and cstring1;
+
+explain
+select count (distinct cint) from alltypesorc where cfloat or cint;
+
+select count (distinct cint) from alltypesorc where cfloat or cint;
+
+SET hive.vectorized.execution.enabled=true;
+
+explain vectorization expression
+select count (distinct cint) from alltypesorc where !cstring1;
+
+select count (distinct cint) from alltypesorc where !cstring1;
+
+explain vectorization expression
+select count (distinct cint) from alltypesorc where cint and cstring1;
+
+select count (distinct cint) from alltypesorc where cint and cstring1;
+
+explain vectorization expression
+select count (distinct cint) from alltypesorc where cfloat or cint;
+
+select count (distinct cint) from alltypesorc where cfloat or cint;
diff --git
a/ql/src/test/results/clientpositive/llap/annotate_stats_filter.q.out
b/ql/src/test/results/clientpositive/llap/annotate_stats_filter.q.out
index fc0cad2..8e1e988 100644
--- a/ql/src/test/results/clientpositive/llap/annotate_stats_filter.q.out
+++ b/ql/src/test/results/clientpositive/llap/annotate_stats_filter.q.out
@@ -375,12 +375,10 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: loc_orc
- Filter Operator
- predicate: 'foo' (type: string)
- Select Operator
- expressions: state (type: string), locid (type: int), zip (type:
bigint), year (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3
- ListSink
+ Select Operator
+ expressions: state (type: string), locid (type: int), zip (type:
bigint), year (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ ListSink
PREHOOK: query: explain select * from loc_orc where true = true
PREHOOK: type: QUERY
diff --git a/ql/src/test/results/clientpositive/llap/vector_empty_where.q.out
b/ql/src/test/results/clientpositive/llap/vector_cast_filter.q.out
similarity index 67%
copy from ql/src/test/results/clientpositive/llap/vector_empty_where.q.out
copy to ql/src/test/results/clientpositive/llap/vector_cast_filter.q.out
index adac463..7d73640 100644
--- a/ql/src/test/results/clientpositive/llap/vector_empty_where.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_cast_filter.q.out
@@ -1,17 +1,13 @@
-PREHOOK: query: explain vectorization expression
-select count (distinct cint) from alltypesorc where cstring1
+PREHOOK: query: explain
+select count (distinct cint) from alltypesorc where !cstring1
PREHOOK: type: QUERY
PREHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-POSTHOOK: query: explain vectorization expression
-select count (distinct cint) from alltypesorc where cstring1
+POSTHOOK: query: explain
+select count (distinct cint) from alltypesorc where !cstring1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-PLAN VECTORIZATION:
- enabled: true
- enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
-
STAGE DEPENDENCIES:
Stage-1 is a root stage
Stage-0 depends on stages: Stage-1
@@ -29,32 +25,16 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: alltypesorc
+ filterExpr: (not UDFToBoolean(cstring1)) (type: boolean)
Statistics: Num rows: 12288 Data size: 899146 Basic stats:
COMPLETE Column stats: COMPLETE
- TableScan Vectorization:
- native: true
Filter Operator
- Filter Vectorization:
- className: VectorFilterOperator
- native: true
- predicateExpression: SelectColumnIsTrue(col
13:boolean)(children: CastStringToBoolean(col 6) -> 13:boolean)
- predicate: cstring1 (type: string)
+ predicate: (not UDFToBoolean(cstring1)) (type: boolean)
Statistics: Num rows: 6144 Data size: 449620 Basic stats:
COMPLETE Column stats: COMPLETE
Select Operator
expressions: cint (type: int)
outputColumnNames: cint
- Select Vectorization:
- className: VectorSelectOperator
- native: true
- projectedOutputColumnNums: [2]
Statistics: Num rows: 6144 Data size: 449620 Basic
stats: COMPLETE Column stats: COMPLETE
Group By Operator
- Group By Vectorization:
- className: VectorGroupByOperator
- groupByMode: HASH
- keyExpressions: col 2:int
- native: false
- vectorProcessingMode: HASH
- projectedOutputColumnNums: []
keys: cint (type: int)
minReductionHashAggr: 0.4
mode: hash
@@ -65,52 +45,19 @@ STAGE PLANS:
null sort order: z
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Reduce Sink Vectorization:
- className: VectorReduceSinkLongOperator
- native: true
- nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
Statistics: Num rows: 6104 Data size: 18228 Basic
stats: COMPLETE Column stats: COMPLETE
- Execution mode: vectorized, llap
+ Execution mode: llap
LLAP IO: all inputs
- Map Vectorization:
- enabled: true
- enabledConditionsMet:
hive.vectorized.use.vectorized.input.format IS true
- inputFormatFeatureSupport: [DECIMAL_64]
- featureSupportInUse: [DECIMAL_64]
- inputFileFormats:
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- allNative: false
- usesVectorUDFAdaptor: false
- vectorized: true
Reducer 2
- Execution mode: vectorized, llap
- Reduce Vectorization:
- enabled: true
- enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez, spark] IS true
- allNative: false
- usesVectorUDFAdaptor: false
- vectorized: true
+ Execution mode: llap
Reduce Operator Tree:
Group By Operator
- Group By Vectorization:
- className: VectorGroupByOperator
- groupByMode: MERGEPARTIAL
- keyExpressions: col 0:int
- native: false
- vectorProcessingMode: MERGE_PARTIAL
- projectedOutputColumnNums: []
keys: KEY._col0 (type: int)
mode: mergepartial
outputColumnNames: _col0
Statistics: Num rows: 6104 Data size: 18228 Basic stats:
COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count(_col0)
- Group By Vectorization:
- aggregators: VectorUDAFCount(col 0:int) -> bigint
- className: VectorGroupByOperator
- groupByMode: HASH
- native: false
- vectorProcessingMode: HASH
- projectedOutputColumnNums: [0]
minReductionHashAggr: 0.99
mode: hash
outputColumnNames: _col0
@@ -118,38 +65,18 @@ STAGE PLANS:
Reduce Output Operator
null sort order:
sort order:
- Reduce Sink Vectorization:
- className: VectorReduceSinkEmptyKeyOperator
- native: true
- nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
value expressions: _col0 (type: bigint)
Reducer 3
- Execution mode: vectorized, llap
- Reduce Vectorization:
- enabled: true
- enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez, spark] IS true
- allNative: false
- usesVectorUDFAdaptor: false
- vectorized: true
+ Execution mode: llap
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
- Group By Vectorization:
- aggregators: VectorUDAFCountMerge(col 0:bigint) -> bigint
- className: VectorGroupByOperator
- groupByMode: MERGEPARTIAL
- native: false
- vectorProcessingMode: GLOBAL
- projectedOutputColumnNums: [0]
mode: mergepartial
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
File Output Operator
compressed: false
- File Sink Vectorization:
- className: VectorFileSinkOperator
- native: false
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
table:
input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
@@ -162,22 +89,222 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: select count (distinct cint) from alltypesorc where cstring1
+PREHOOK: query: select count (distinct cint) from alltypesorc where !cstring1
PREHOOK: type: QUERY
PREHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-POSTHOOK: query: select count (distinct cint) from alltypesorc where cstring1
+POSTHOOK: query: select count (distinct cint) from alltypesorc where !cstring1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+0
+PREHOOK: query: explain
+select count (distinct cint) from alltypesorc where cint and cstring1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: explain
+select count (distinct cint) from alltypesorc where cint and cstring1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypesorc
+ filterExpr: (UDFToBoolean(cint) and UDFToBoolean(cstring1))
(type: boolean)
+ Statistics: Num rows: 12288 Data size: 899146 Basic stats:
COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: (UDFToBoolean(cint) and UDFToBoolean(cstring1))
(type: boolean)
+ Statistics: Num rows: 3072 Data size: 224812 Basic stats:
COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: cint (type: int)
+ outputColumnNames: cint
+ Statistics: Num rows: 3072 Data size: 224812 Basic
stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ keys: cint (type: int)
+ minReductionHashAggr: 0.4
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 3072 Data size: 9176 Basic
stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 3072 Data size: 9176 Basic
stats: COMPLETE Column stats: COMPLETE
+ Execution mode: llap
+ LLAP IO: all inputs
+ Reducer 2
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ keys: KEY._col0 (type: int)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 3072 Data size: 9176 Basic stats:
COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count(_col0)
+ minReductionHashAggr: 0.99
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ Reduce Output Operator
+ null sort order:
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ value expressions: _col0 (type: bigint)
+ Reducer 3
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count (distinct cint) from alltypesorc where cint and
cstring1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: select count (distinct cint) from alltypesorc where cint and
cstring1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
6041
+PREHOOK: query: explain
+select count (distinct cint) from alltypesorc where cfloat or cint
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: explain
+select count (distinct cint) from alltypesorc where cfloat or cint
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypesorc
+ filterExpr: (UDFToBoolean(cfloat) or UDFToBoolean(cint))
(type: boolean)
+ Statistics: Num rows: 12288 Data size: 73392 Basic stats:
COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: (UDFToBoolean(cfloat) or UDFToBoolean(cint))
(type: boolean)
+ Statistics: Num rows: 12288 Data size: 73392 Basic stats:
COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: cint (type: int)
+ outputColumnNames: cint
+ Statistics: Num rows: 12288 Data size: 73392 Basic
stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ keys: cint (type: int)
+ minReductionHashAggr: 0.5031738
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 6105 Data size: 18232 Basic
stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 6105 Data size: 18232 Basic
stats: COMPLETE Column stats: COMPLETE
+ Execution mode: llap
+ LLAP IO: all inputs
+ Reducer 2
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ keys: KEY._col0 (type: int)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 6105 Data size: 18232 Basic stats:
COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count(_col0)
+ minReductionHashAggr: 0.99
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ Reduce Output Operator
+ null sort order:
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ value expressions: _col0 (type: bigint)
+ Reducer 3
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count (distinct cint) from alltypesorc where cfloat or
cint
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: select count (distinct cint) from alltypesorc where cfloat or
cint
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+6082
PREHOOK: query: explain vectorization expression
-select count (distinct cint) from alltypesorc where cint
+select count (distinct cint) from alltypesorc where !cstring1
PREHOOK: type: QUERY
PREHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
POSTHOOK: query: explain vectorization expression
-select count (distinct cint) from alltypesorc where cint
+select count (distinct cint) from alltypesorc where !cstring1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
@@ -202,39 +329,48 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: alltypesorc
- Statistics: Num rows: 12288 Data size: 36696 Basic stats:
COMPLETE Column stats: COMPLETE
+ filterExpr: (not UDFToBoolean(cstring1)) (type: boolean)
+ Statistics: Num rows: 12288 Data size: 899146 Basic stats:
COMPLETE Column stats: COMPLETE
TableScan Vectorization:
native: true
Filter Operator
Filter Vectorization:
className: VectorFilterOperator
native: true
- predicateExpression: SelectColumnIsTrue(col
13:boolean)(children: CastLongToBooleanViaLongToLong(col 2:int) -> 13:boolean)
- predicate: cint (type: int)
- Statistics: Num rows: 6144 Data size: 18348 Basic stats:
COMPLETE Column stats: COMPLETE
- Group By Operator
- Group By Vectorization:
- className: VectorGroupByOperator
- groupByMode: HASH
- keyExpressions: col 2:int
- native: false
- vectorProcessingMode: HASH
- projectedOutputColumnNums: []
- keys: cint (type: int)
- minReductionHashAggr: 0.5030924
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 3053 Data size: 9120 Basic stats:
COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- null sort order: z
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Reduce Sink Vectorization:
- className: VectorReduceSinkLongOperator
- native: true
- nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 3053 Data size: 9120 Basic
stats: COMPLETE Column stats: COMPLETE
+ predicateExpression: SelectColumnIsFalse(col
13:boolean)(children: CastStringToBoolean(col 6) -> 13:boolean)
+ predicate: (not UDFToBoolean(cstring1)) (type: boolean)
+ Statistics: Num rows: 6144 Data size: 449620 Basic stats:
COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: cint (type: int)
+ outputColumnNames: cint
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [2]
+ Statistics: Num rows: 6144 Data size: 449620 Basic
stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ Group By Vectorization:
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ keyExpressions: col 2:int
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumnNums: []
+ keys: cint (type: int)
+ minReductionHashAggr: 0.4
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 6104 Data size: 18228 Basic
stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkLongOperator
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ Statistics: Num rows: 6104 Data size: 18228 Basic
stats: COMPLETE Column stats: COMPLETE
Execution mode: vectorized, llap
LLAP IO: all inputs
Map Vectorization:
@@ -266,7 +402,7 @@ STAGE PLANS:
keys: KEY._col0 (type: int)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 3053 Data size: 9120 Basic stats:
COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 6104 Data size: 18228 Basic stats:
COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count(_col0)
Group By Vectorization:
@@ -327,22 +463,22 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: select count (distinct cint) from alltypesorc where cint
+PREHOOK: query: select count (distinct cint) from alltypesorc where !cstring1
PREHOOK: type: QUERY
PREHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-POSTHOOK: query: select count (distinct cint) from alltypesorc where cint
+POSTHOOK: query: select count (distinct cint) from alltypesorc where !cstring1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-6082
+0
PREHOOK: query: explain vectorization expression
-select count (distinct cint) from alltypesorc where cfloat
+select count (distinct cint) from alltypesorc where cint and cstring1
PREHOOK: type: QUERY
PREHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
POSTHOOK: query: explain vectorization expression
-select count (distinct cint) from alltypesorc where cfloat
+select count (distinct cint) from alltypesorc where cint and cstring1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
@@ -367,16 +503,17 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: alltypesorc
- Statistics: Num rows: 12288 Data size: 73392 Basic stats:
COMPLETE Column stats: COMPLETE
+ filterExpr: (UDFToBoolean(cint) and UDFToBoolean(cstring1))
(type: boolean)
+ Statistics: Num rows: 12288 Data size: 899146 Basic stats:
COMPLETE Column stats: COMPLETE
TableScan Vectorization:
native: true
Filter Operator
Filter Vectorization:
className: VectorFilterOperator
native: true
- predicateExpression: SelectColumnIsTrue(col
13:boolean)(children: CastDoubleToBooleanViaDoubleToLong(col 4:float) ->
13:boolean)
- predicate: cfloat (type: float)
- Statistics: Num rows: 6144 Data size: 36696 Basic stats:
COMPLETE Column stats: COMPLETE
+ predicateExpression: FilterExprAndExpr(children:
SelectColumnIsTrue(col 13:boolean)(children: CastLongToBooleanViaLongToLong(col
2:int) -> 13:boolean), SelectColumnIsTrue(col 14:boolean)(children:
CastStringToBoolean(col 6) -> 14:boolean))
+ predicate: (UDFToBoolean(cint) and UDFToBoolean(cstring1))
(type: boolean)
+ Statistics: Num rows: 3072 Data size: 224812 Basic stats:
COMPLETE Column stats: COMPLETE
Select Operator
expressions: cint (type: int)
outputColumnNames: cint
@@ -384,7 +521,7 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [2]
- Statistics: Num rows: 6144 Data size: 36696 Basic stats:
COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3072 Data size: 224812 Basic
stats: COMPLETE Column stats: COMPLETE
Group By Operator
Group By Vectorization:
className: VectorGroupByOperator
@@ -397,7 +534,7 @@ STAGE PLANS:
minReductionHashAggr: 0.4
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 6104 Data size: 18228 Basic
stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3072 Data size: 9176 Basic
stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
null sort order: z
@@ -407,7 +544,7 @@ STAGE PLANS:
className: VectorReduceSinkLongOperator
native: true
nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 6104 Data size: 18228 Basic
stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3072 Data size: 9176 Basic
stats: COMPLETE Column stats: COMPLETE
Execution mode: vectorized, llap
LLAP IO: all inputs
Map Vectorization:
@@ -439,7 +576,7 @@ STAGE PLANS:
keys: KEY._col0 (type: int)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 6104 Data size: 18228 Basic stats:
COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3072 Data size: 9176 Basic stats:
COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count(_col0)
Group By Vectorization:
@@ -500,22 +637,22 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: select count (distinct cint) from alltypesorc where cfloat
+PREHOOK: query: select count (distinct cint) from alltypesorc where cint and
cstring1
PREHOOK: type: QUERY
PREHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-POSTHOOK: query: select count (distinct cint) from alltypesorc where cfloat
+POSTHOOK: query: select count (distinct cint) from alltypesorc where cint and
cstring1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-3022
+6041
PREHOOK: query: explain vectorization expression
-select count (distinct cint) from alltypesorc where ctimestamp1
+select count (distinct cint) from alltypesorc where cfloat or cint
PREHOOK: type: QUERY
PREHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
POSTHOOK: query: explain vectorization expression
-select count (distinct cint) from alltypesorc where ctimestamp1
+select count (distinct cint) from alltypesorc where cfloat or cint
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
@@ -540,16 +677,17 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: alltypesorc
- Statistics: Num rows: 12288 Data size: 403656 Basic stats:
COMPLETE Column stats: COMPLETE
+ filterExpr: (UDFToBoolean(cfloat) or UDFToBoolean(cint))
(type: boolean)
+ Statistics: Num rows: 12288 Data size: 73392 Basic stats:
COMPLETE Column stats: COMPLETE
TableScan Vectorization:
native: true
Filter Operator
Filter Vectorization:
className: VectorFilterOperator
native: true
- predicateExpression: SelectColumnIsTrue(col
13:boolean)(children: CastTimestampToBoolean(col 8:timestamp) -> 13:boolean)
- predicate: ctimestamp1 (type: timestamp)
- Statistics: Num rows: 6144 Data size: 201828 Basic stats:
COMPLETE Column stats: COMPLETE
+ predicateExpression: FilterExprOrExpr(children:
SelectColumnIsTrue(col 13:boolean)(children:
CastDoubleToBooleanViaDoubleToLong(col 4:float) -> 13:boolean),
SelectColumnIsTrue(col 14:boolean)(children: CastLongToBooleanViaLongToLong(col
2:int) -> 14:boolean))
+ predicate: (UDFToBoolean(cfloat) or UDFToBoolean(cint))
(type: boolean)
+ Statistics: Num rows: 12288 Data size: 73392 Basic stats:
COMPLETE Column stats: COMPLETE
Select Operator
expressions: cint (type: int)
outputColumnNames: cint
@@ -557,7 +695,7 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [2]
- Statistics: Num rows: 6144 Data size: 201828 Basic
stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 12288 Data size: 73392 Basic
stats: COMPLETE Column stats: COMPLETE
Group By Operator
Group By Vectorization:
className: VectorGroupByOperator
@@ -567,10 +705,10 @@ STAGE PLANS:
vectorProcessingMode: HASH
projectedOutputColumnNums: []
keys: cint (type: int)
- minReductionHashAggr: 0.4
+ minReductionHashAggr: 0.5031738
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 6104 Data size: 18228 Basic
stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 6105 Data size: 18232 Basic
stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
null sort order: z
@@ -580,7 +718,7 @@ STAGE PLANS:
className: VectorReduceSinkLongOperator
native: true
nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 6104 Data size: 18228 Basic
stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 6105 Data size: 18232 Basic
stats: COMPLETE Column stats: COMPLETE
Execution mode: vectorized, llap
LLAP IO: all inputs
Map Vectorization:
@@ -612,7 +750,7 @@ STAGE PLANS:
keys: KEY._col0 (type: int)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 6104 Data size: 18228 Basic stats:
COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 6105 Data size: 18232 Basic stats:
COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count(_col0)
Group By Vectorization:
@@ -673,12 +811,12 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: select count (distinct cint) from alltypesorc where ctimestamp1
+PREHOOK: query: select count (distinct cint) from alltypesorc where cfloat or
cint
PREHOOK: type: QUERY
PREHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-POSTHOOK: query: select count (distinct cint) from alltypesorc where
ctimestamp1
+POSTHOOK: query: select count (distinct cint) from alltypesorc where cfloat or
cint
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-3022
+6082
diff --git a/ql/src/test/results/clientpositive/llap/vector_empty_where.q.out
b/ql/src/test/results/clientpositive/llap/vector_empty_where.q.out
index adac463..218fc41 100644
--- a/ql/src/test/results/clientpositive/llap/vector_empty_where.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_empty_where.q.out
@@ -29,6 +29,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: alltypesorc
+ filterExpr: UDFToBoolean(cstring1) (type: boolean)
Statistics: Num rows: 12288 Data size: 899146 Basic stats:
COMPLETE Column stats: COMPLETE
TableScan Vectorization:
native: true
@@ -37,7 +38,7 @@ STAGE PLANS:
className: VectorFilterOperator
native: true
predicateExpression: SelectColumnIsTrue(col
13:boolean)(children: CastStringToBoolean(col 6) -> 13:boolean)
- predicate: cstring1 (type: string)
+ predicate: UDFToBoolean(cstring1) (type: boolean)
Statistics: Num rows: 6144 Data size: 449620 Basic stats:
COMPLETE Column stats: COMPLETE
Select Operator
expressions: cint (type: int)
@@ -202,6 +203,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: alltypesorc
+ filterExpr: UDFToBoolean(cint) (type: boolean)
Statistics: Num rows: 12288 Data size: 36696 Basic stats:
COMPLETE Column stats: COMPLETE
TableScan Vectorization:
native: true
@@ -210,7 +212,7 @@ STAGE PLANS:
className: VectorFilterOperator
native: true
predicateExpression: SelectColumnIsTrue(col
13:boolean)(children: CastLongToBooleanViaLongToLong(col 2:int) -> 13:boolean)
- predicate: cint (type: int)
+ predicate: UDFToBoolean(cint) (type: boolean)
Statistics: Num rows: 6144 Data size: 18348 Basic stats:
COMPLETE Column stats: COMPLETE
Group By Operator
Group By Vectorization:
@@ -221,10 +223,10 @@ STAGE PLANS:
vectorProcessingMode: HASH
projectedOutputColumnNums: []
keys: cint (type: int)
- minReductionHashAggr: 0.5030924
+ minReductionHashAggr: 0.4
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 3053 Data size: 9120 Basic stats:
COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 6104 Data size: 18228 Basic stats:
COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
null sort order: z
@@ -234,7 +236,7 @@ STAGE PLANS:
className: VectorReduceSinkLongOperator
native: true
nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 3053 Data size: 9120 Basic
stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 6104 Data size: 18228 Basic
stats: COMPLETE Column stats: COMPLETE
Execution mode: vectorized, llap
LLAP IO: all inputs
Map Vectorization:
@@ -266,7 +268,7 @@ STAGE PLANS:
keys: KEY._col0 (type: int)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 3053 Data size: 9120 Basic stats:
COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 6104 Data size: 18228 Basic stats:
COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count(_col0)
Group By Vectorization:
@@ -367,6 +369,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: alltypesorc
+ filterExpr: UDFToBoolean(cfloat) (type: boolean)
Statistics: Num rows: 12288 Data size: 73392 Basic stats:
COMPLETE Column stats: COMPLETE
TableScan Vectorization:
native: true
@@ -375,7 +378,7 @@ STAGE PLANS:
className: VectorFilterOperator
native: true
predicateExpression: SelectColumnIsTrue(col
13:boolean)(children: CastDoubleToBooleanViaDoubleToLong(col 4:float) ->
13:boolean)
- predicate: cfloat (type: float)
+ predicate: UDFToBoolean(cfloat) (type: boolean)
Statistics: Num rows: 6144 Data size: 36696 Basic stats:
COMPLETE Column stats: COMPLETE
Select Operator
expressions: cint (type: int)
@@ -540,6 +543,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: alltypesorc
+ filterExpr: UDFToBoolean(ctimestamp1) (type: boolean)
Statistics: Num rows: 12288 Data size: 403656 Basic stats:
COMPLETE Column stats: COMPLETE
TableScan Vectorization:
native: true
@@ -548,7 +552,7 @@ STAGE PLANS:
className: VectorFilterOperator
native: true
predicateExpression: SelectColumnIsTrue(col
13:boolean)(children: CastTimestampToBoolean(col 8:timestamp) -> 13:boolean)
- predicate: ctimestamp1 (type: timestamp)
+ predicate: UDFToBoolean(ctimestamp1) (type: boolean)
Statistics: Num rows: 6144 Data size: 201828 Basic stats:
COMPLETE Column stats: COMPLETE
Select Operator
expressions: cint (type: int)
