This is an automated email from the ASF dual-hosted git repository.
deniskuzZ pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 96e339057df HIVE-25125: PTF: Vectorize percent_rank function (#6752)
96e339057df is described below
commit 96e339057df05dbe572729cd185f263f377a6c58
Author: ramitg254 <[email protected]>
AuthorDate: Tue Sep 22 13:35:28 2026 +0530
HIVE-25125: PTF: Vectorize percent_rank function (#6752)
---
.../vector/ptf/VectorPTFEvaluatorPercentRank.java | 98 ++
.../hive/ql/optimizer/physical/Vectorizer.java | 75 +-
.../apache/hadoop/hive/ql/plan/VectorPTFDesc.java | 15 +
.../clientpositive/vector_ptf_percent_rank.q | 113 ++
.../vector_ptf_unbuffered_partition_col.q | 42 +
.../llap/vector_ptf_percent_rank.q.out | 1176 ++++++++++++++++++++
.../llap/vector_ptf_unbuffered_partition_col.q.out | 1059 ++++++++++++++++++
.../clientpositive/llap/vector_windowing.q.out | 2 +-
.../llap/vector_windowing_gby2.q.out | 39 +-
.../llap/vector_windowing_rank.q.out | 42 +-
.../clientpositive/llap/windowing_gby2.q.out | 2 +-
11 files changed, 2651 insertions(+), 12 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorPercentRank.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorPercentRank.java
new file mode 100644
index 00000000000..6a8b9d07716
--- /dev/null
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorPercentRank.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.ptf;
+
+import java.util.List;
+
+import org.apache.hadoop.hive.ql.exec.vector.ColumnVector.Type;
+import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.ptf.WindowFrameDef;
+
+/**
+ * Evaluates {@code percent_rank()} as a <b>group-aggregated streaming</b>
+ * evaluator.
+ *
+ * <p>
+ * The partition is buffered so {@link #setPartitionSize(int)} is known before
+ * output is written.
+ * Peer-group rank is tracked incrementally (like
+ * {@link VectorPTFEvaluatorRank}) during batch
+ * forward; {@link #addStreamingGroupResults} is a no-op because no pre-pass is
+ * required.
+ */
+public class VectorPTFEvaluatorPercentRank extends VectorPTFEvaluatorBase {
+
+ private int rank;
+ private int groupCount;
+
+ public VectorPTFEvaluatorPercentRank(WindowFrameDef windowFrameDef, int
outputColumnNum) {
+ super(windowFrameDef, outputColumnNum);
+ resetEvaluator();
+ }
+
+ @Override
+ public boolean isGroupAggregatedStreamingEvaluator() {
+ return true;
+ }
+
+ @Override
+ public void addStreamingGroupResults(List<Integer> groupRowCounts) {
+ // No-op: percent_rank only needs the partition size and the current rank.
+ }
+
+ @Override
+ public void evaluateGroupBatch(VectorizedRowBatch batch) throws
HiveException {
+ if (partitionSize <= 0) {
+ throw new HiveException("Partition size must be set before computing
percent_rank");
+ }
+ final double divisor = partitionSize > 1 ? partitionSize - 1 : 1;
+ DoubleColumnVector outputColVector = (DoubleColumnVector)
batch.cols[outputColumnNum];
+ outputColVector.isRepeating = true;
+ outputColVector.noNulls = true;
+ outputColVector.isNull[0] = false;
+ outputColVector.vector[0] = (rank - 1) / divisor;
+ groupCount += batch.size;
+ }
+
+ @Override
+ public void doLastBatchWork() {
+ rank += groupCount;
+ groupCount = 0;
+ }
+
+ @Override
+ public boolean streamsResult() {
+ return true;
+ }
+
+ @Override
+ public Type getResultColumnVectorType() {
+ return Type.DOUBLE;
+ }
+
+ @Override
+ public void resetEvaluator() {
+ rank = 1;
+ partitionSize = -1;
+ groupCount = 0;
+ }
+}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
index 00cdcc50ad1..36cc904c261 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
@@ -2931,6 +2931,12 @@ private boolean validatePTFOperator(PTFOperator op,
VectorizationContext vContex
return false;
}
+ if (hasUnbufferedPartitionColumnInEvaluatorArgs(vectorPTFDesc)) {
+ setOperatorIssue(
+ "Window function argument references partition-only column not
buffered in vector PTF");
+ return false;
+ }
+
// Output columns ok?
String[] outputColumnNames = vectorPTFDesc.getOutputColumnNames();
TypeInfo[] outputTypeInfos = vectorPTFDesc.getOutputTypeInfos();
@@ -3005,10 +3011,7 @@ private boolean validatePTFOperator(PTFOperator op,
VectorizationContext vContex
throw new RuntimeException("Unexpected window type " +
windowFrameDef.getWindowType());
}
- // RANK/DENSE_RANK/CUME_DIST don't care about columns.
- if (supportedFunctionType != SupportedFunctionType.RANK &&
- supportedFunctionType != SupportedFunctionType.DENSE_RANK &&
- supportedFunctionType != SupportedFunctionType.CUME_DIST) {
+ if
(!VectorPTFDesc.COLUMN_AGNOSTIC_FUNCTIONS.contains(supportedFunctionType)) {
if (exprNodeDescList != null) {
// LEAD and LAG now supports multiple arguments in vectorized mode
@@ -5039,6 +5042,70 @@ private static ExprNodeDesc[]
getOrderExprNodeDescs(List<OrderExpressionDef> ord
return exprNodeDescs;
}
+ // TODO: An evaluator that wants to handle an unbuffered partition-only
column in its calculation could
+ // opt in to vectorization here.
+ private static boolean hasUnbufferedPartitionColumnInEvaluatorArgs(
+ VectorPTFDesc vectorPTFDesc) {
+
+ // PARTITION BY matches ORDER BY, so partition cols are buffered as order
cols.
+ if (!vectorPTFDesc.getIsPartitionOrderBy()) {
+ return false;
+ }
+
+ List<ExprNodeDesc> partitionOnlyExprs =
getPartitionOnlyExprs(vectorPTFDesc.getPartitionExprNodeDescs(),
+ vectorPTFDesc.getOrderExprNodeDescs());
+ if (partitionOnlyExprs.isEmpty()) {
+ return false;
+ }
+
+ return evaluatorArgsReferencePartitionOnlyExprs(
+ vectorPTFDesc.getEvaluatorFunctionNames(),
vectorPTFDesc.getEvaluatorInputExprNodeDescLists(),
+ partitionOnlyExprs);
+ }
+
+ private static boolean evaluatorArgsReferencePartitionOnlyExprs(
+ String[] evaluatorFunctionNames,
+ List<ExprNodeDesc>[] evaluatorInputExprNodeDescLists,
+ List<ExprNodeDesc> partitionOnlyExprs) {
+ for (int i = 0; i < evaluatorFunctionNames.length; i++) {
+ SupportedFunctionType supportedFunctionType =
+
VectorPTFDesc.supportedFunctionsMap.get(evaluatorFunctionNames[i].toLowerCase());
+ List<ExprNodeDesc> exprNodeDescList = evaluatorInputExprNodeDescLists[i];
+ if (supportedFunctionType == null ||
+
VectorPTFDesc.COLUMN_AGNOSTIC_FUNCTIONS.contains(supportedFunctionType) ||
+ exprNodeDescList == null) {
+ continue;
+ }
+
+ // Check whether an evaluator argument references a partition-only
column.
+ if (exprNodeDescList.stream()
+ .anyMatch(expr -> hasPartitionOnlyColumnArg(expr,
partitionOnlyExprs))) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static boolean hasPartitionOnlyColumnArg(
+ ExprNodeDesc expr, List<ExprNodeDesc> partitionOnlyExprs) {
+ if (!(expr instanceof ExprNodeColumnDesc)) {
+ return false;
+ }
+ return partitionOnlyExprs.stream().anyMatch(expr::isSame);
+ }
+
+ private static List<ExprNodeDesc> getPartitionOnlyExprs(
+ ExprNodeDesc[] partitionExprNodeDescs, ExprNodeDesc[]
orderExprNodeDescs) {
+ List<ExprNodeDesc> partitionOnlyExprs = new ArrayList<ExprNodeDesc>();
+ for (ExprNodeDesc partitionExpr : partitionExprNodeDescs) {
+ // Collect partition expressions that are not also ORDER BY expressions.
+ if (Arrays.stream(orderExprNodeDescs).noneMatch(partitionExpr::isSame)) {
+ partitionOnlyExprs.add(partitionExpr);
+ }
+ }
+ return partitionOnlyExprs;
+ }
+
/*
* Update the VectorPTFDesc with data that is used during validation and
that doesn't rely on
* VectorizationContext to lookup column names, etc.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorPTFDesc.java
b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorPTFDesc.java
index aab04bee56c..38f730049dd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorPTFDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorPTFDesc.java
@@ -20,8 +20,10 @@
package org.apache.hadoop.hive.ql.plan;
import java.util.ArrayList;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
+import java.util.Set;
import java.util.TreeSet;
import org.apache.commons.lang3.ArrayUtils;
@@ -58,6 +60,7 @@
import org.apache.hadoop.hive.ql.exec.vector.ptf.VectorPTFEvaluatorLongMax;
import org.apache.hadoop.hive.ql.exec.vector.ptf.VectorPTFEvaluatorLongMin;
import org.apache.hadoop.hive.ql.exec.vector.ptf.VectorPTFEvaluatorLongSum;
+import org.apache.hadoop.hive.ql.exec.vector.ptf.VectorPTFEvaluatorPercentRank;
import org.apache.hadoop.hive.ql.exec.vector.ptf.VectorPTFEvaluatorRank;
import org.apache.hadoop.hive.ql.exec.vector.ptf.VectorPTFEvaluatorRowNumber;
import
org.apache.hadoop.hive.ql.exec.vector.ptf.VectorPTFEvaluatorStreamingDecimalAvg;
@@ -93,6 +96,7 @@ public enum SupportedFunctionType {
ROW_NUMBER,
RANK,
DENSE_RANK,
+ PERCENT_RANK,
CUME_DIST,
MIN,
MAX,
@@ -133,6 +137,14 @@ public boolean isSupportDistinct() {
supportedFunctionNames.addAll(treeSet);
}
+ // Functions that do not depend on input columns.
+ public static final Set<SupportedFunctionType> COLUMN_AGNOSTIC_FUNCTIONS =
+ EnumSet.of(
+ SupportedFunctionType.RANK,
+ SupportedFunctionType.DENSE_RANK,
+ SupportedFunctionType.PERCENT_RANK,
+ SupportedFunctionType.CUME_DIST);
+
private TypeInfo[] reducerBatchTypeInfos;
private DataTypePhysicalVariation[] reducerBatchDataTypePhysicalVariations;
@@ -204,6 +216,9 @@ public static VectorPTFEvaluatorBase
getEvaluator(SupportedFunctionType function
case DENSE_RANK:
evaluator = new VectorPTFEvaluatorDenseRank(windowFrameDef,
outputColumnNum);
break;
+ case PERCENT_RANK:
+ evaluator = new VectorPTFEvaluatorPercentRank(windowFrameDef,
outputColumnNum);
+ break;
case CUME_DIST:
evaluator = new VectorPTFEvaluatorCumeDist(windowFrameDef,
outputColumnNum);
break;
diff --git a/ql/src/test/queries/clientpositive/vector_ptf_percent_rank.q
b/ql/src/test/queries/clientpositive/vector_ptf_percent_rank.q
new file mode 100644
index 00000000000..f9316793ea8
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/vector_ptf_percent_rank.q
@@ -0,0 +1,113 @@
+set hive.vectorized.testing.reducer.batch.size=2;
+
+DROP TABLE IF EXISTS vector_ptf_percent_rank_int;
+
+CREATE TABLE vector_ptf_percent_rank_int(name string, rowindex int, mynumber
int) stored as orc;
+
+INSERT INTO vector_ptf_percent_rank_int values
+('five', 1, 10),
+('five', 2, 20),
+('five', 3, 30),
+('five', 4, 40),
+('five', 5, 50),
+('six', 1, 10),
+('six', 2, 20),
+('six', 3, 30),
+('six', 4, 40),
+('six', 5, 50),
+('six', 6, 60),
+-- single-row partition: percent_rank 0.0
+('lonely', 99, 42),
+-- two-row null partition
+(NULL, 1, 100),
+(NULL, 2, 100);
+
+-- NON-VECTORIZED
+set hive.vectorized.execution.ptf.enabled=false;
+
+select name, rowindex, mynumber,
+percent_rank() over (partition by name order by mynumber) as pr
+from vector_ptf_percent_rank_int;
+
+select name, rowindex, mynumber,
+rank() over (partition by name order by mynumber) as r,
+dense_rank() over (partition by name order by mynumber) as dr,
+percent_rank() over (partition by name order by mynumber) as pr
+from vector_ptf_percent_rank_int;
+
+select name, rowindex, mynumber,
+rank() over (order by mynumber) as r,
+dense_rank() over (order by mynumber) as dr,
+percent_rank() over (order by mynumber) as pr
+from vector_ptf_percent_rank_int;
+
+select name, rowindex, mynumber,
+rank() over (partition by name) as r,
+dense_rank() over (partition by name) as dr,
+percent_rank() over (partition by name) as pr
+from vector_ptf_percent_rank_int;
+
+select name, rowindex, mynumber,
+rank() over () as r,
+dense_rank() over () as dr,
+percent_rank() over () as pr
+from vector_ptf_percent_rank_int;
+
+-- VECTORIZED
+set hive.vectorized.execution.ptf.enabled=true;
+
+explain vectorization detail select name, rowindex, mynumber,
+percent_rank() over (partition by name order by mynumber) as pr
+from vector_ptf_percent_rank_int;
+
+select name, rowindex, mynumber,
+percent_rank() over (partition by name order by mynumber) as pr
+from vector_ptf_percent_rank_int;
+
+explain vectorization detail select name, rowindex, mynumber,
+rank() over (partition by name order by mynumber) as r,
+dense_rank() over (partition by name order by mynumber) as dr,
+percent_rank() over (partition by name order by mynumber) as pr
+from vector_ptf_percent_rank_int;
+
+select name, rowindex, mynumber,
+rank() over (partition by name order by mynumber) as r,
+dense_rank() over (partition by name order by mynumber) as dr,
+percent_rank() over (partition by name order by mynumber) as pr
+from vector_ptf_percent_rank_int;
+
+explain vectorization detail select name, rowindex, mynumber,
+rank() over (order by mynumber) as r,
+dense_rank() over (order by mynumber) as dr,
+percent_rank() over (order by mynumber) as pr
+from vector_ptf_percent_rank_int;
+
+select name, rowindex, mynumber,
+rank() over (order by mynumber) as r,
+dense_rank() over (order by mynumber) as dr,
+percent_rank() over (order by mynumber) as pr
+from vector_ptf_percent_rank_int;
+
+explain vectorization detail select name, rowindex, mynumber,
+rank() over (partition by name) as r,
+dense_rank() over (partition by name) as dr,
+percent_rank() over (partition by name) as pr
+from vector_ptf_percent_rank_int;
+
+select name, rowindex, mynumber,
+rank() over (partition by name) as r,
+dense_rank() over (partition by name) as dr,
+percent_rank() over (partition by name) as pr
+from vector_ptf_percent_rank_int;
+
+explain vectorization detail select name, rowindex, mynumber,
+rank() over () as r,
+dense_rank() over () as dr,
+percent_rank() over () as pr
+from vector_ptf_percent_rank_int;
+
+select name, rowindex, mynumber,
+rank() over () as r,
+dense_rank() over () as dr,
+percent_rank() over () as pr
+from vector_ptf_percent_rank_int;
diff --git
a/ql/src/test/queries/clientpositive/vector_ptf_unbuffered_partition_col.q
b/ql/src/test/queries/clientpositive/vector_ptf_unbuffered_partition_col.q
new file mode 100644
index 00000000000..1b484e7fbb1
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/vector_ptf_unbuffered_partition_col.q
@@ -0,0 +1,42 @@
+--! qt:dataset:cbo_t1
+
+-- Case 1: PARTITION BY and ORDER BY are identical (vectorized).
+explain vectorization detail
+select key, sum(c_float) over(partition by c_float order by c_float) as s
+from cbo_t1 order by key, s;
+
+select key, sum(c_float) over(partition by c_float order by c_float) as s
+from cbo_t1 order by key, s;
+
+-- Case 2: multiple partition columns with partial ORDER BY overlap; arg
avoids partition-only cols (vectorized).
+explain vectorization detail
+select key, sum(c_float) over(partition by c_float, key order by c_float,
value) as s
+from cbo_t1 order by key, s;
+
+select key, sum(c_float) over(partition by c_float, key order by c_float,
value) as s
+from cbo_t1 order by key, s;
+
+-- Case 3: partition-only column referenced directly as a column argument
(row-mode fallback).
+explain vectorization detail
+select key, sum(c_float) over(partition by c_float order by key) as s
+from cbo_t1 order by key, s;
+
+select key, sum(c_float) over(partition by c_float order by key) as s
+from cbo_t1 order by key, s;
+
+-- Case 4: partition-only column inside an expression argument (vectorized).
+explain vectorization detail
+select key, sum(c_float + 1) over(partition by c_float order by key) as s
+from cbo_t1 order by key, s;
+
+select key, sum(c_float + 1) over(partition by c_float order by key) as s
+from cbo_t1 order by key, s;
+
+-- Case 5: PARTITION BY expression; direct column arg is a leaf of that expr,
not the partition expr (vectorized).
+explain vectorization detail
+select key, sum(c_float) over(partition by c_float + 1 order by key) as s
+from cbo_t1 order by key, s;
+
+select key, sum(c_float) over(partition by c_float + 1 order by key) as s
+from cbo_t1 order by key, s;
+
diff --git
a/ql/src/test/results/clientpositive/llap/vector_ptf_percent_rank.q.out
b/ql/src/test/results/clientpositive/llap/vector_ptf_percent_rank.q.out
new file mode 100644
index 00000000000..cd3ccd4f5df
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/vector_ptf_percent_rank.q.out
@@ -0,0 +1,1176 @@
+PREHOOK: query: DROP TABLE IF EXISTS vector_ptf_percent_rank_int
+PREHOOK: type: DROPTABLE
+PREHOOK: Output: database:default
+POSTHOOK: query: DROP TABLE IF EXISTS vector_ptf_percent_rank_int
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: database:default
+PREHOOK: query: CREATE TABLE vector_ptf_percent_rank_int(name string, rowindex
int, mynumber int) stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@vector_ptf_percent_rank_int
+POSTHOOK: query: CREATE TABLE vector_ptf_percent_rank_int(name string,
rowindex int, mynumber int) stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@vector_ptf_percent_rank_int
+PREHOOK: query: INSERT INTO vector_ptf_percent_rank_int values
+('five', 1, 10),
+('five', 2, 20),
+('five', 3, 30),
+('five', 4, 40),
+('five', 5, 50),
+('six', 1, 10),
+('six', 2, 20),
+('six', 3, 30),
+('six', 4, 40),
+('six', 5, 50),
+('six', 6, 60),
+
+('lonely', 99, 42),
+
+(NULL, 1, 100),
+(NULL, 2, 100)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@vector_ptf_percent_rank_int
+POSTHOOK: query: INSERT INTO vector_ptf_percent_rank_int values
+('five', 1, 10),
+('five', 2, 20),
+('five', 3, 30),
+('five', 4, 40),
+('five', 5, 50),
+('six', 1, 10),
+('six', 2, 20),
+('six', 3, 30),
+('six', 4, 40),
+('six', 5, 50),
+('six', 6, 60),
+
+('lonely', 99, 42),
+
+(NULL, 1, 100),
+(NULL, 2, 100)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@vector_ptf_percent_rank_int
+POSTHOOK: Lineage: vector_ptf_percent_rank_int.mynumber SCRIPT []
+POSTHOOK: Lineage: vector_ptf_percent_rank_int.name SCRIPT []
+POSTHOOK: Lineage: vector_ptf_percent_rank_int.rowindex SCRIPT []
+PREHOOK: query: select name, rowindex, mynumber,
+percent_rank() over (partition by name order by mynumber) as pr
+from vector_ptf_percent_rank_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+percent_rank() over (partition by name order by mynumber) as pr
+from vector_ptf_percent_rank_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+NULL 2 100 0.0
+NULL 1 100 0.0
+five 1 10 0.0
+five 2 20 0.25
+five 3 30 0.5
+five 4 40 0.75
+five 5 50 1.0
+lonely 99 42 0.0
+six 1 10 0.0
+six 2 20 0.2
+six 3 30 0.4
+six 4 40 0.6
+six 5 50 0.8
+six 6 60 1.0
+PREHOOK: query: select name, rowindex, mynumber,
+rank() over (partition by name order by mynumber) as r,
+dense_rank() over (partition by name order by mynumber) as dr,
+percent_rank() over (partition by name order by mynumber) as pr
+from vector_ptf_percent_rank_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+rank() over (partition by name order by mynumber) as r,
+dense_rank() over (partition by name order by mynumber) as dr,
+percent_rank() over (partition by name order by mynumber) as pr
+from vector_ptf_percent_rank_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+NULL 2 100 1 1 0.0
+NULL 1 100 1 1 0.0
+five 1 10 1 1 0.0
+five 2 20 2 2 0.25
+five 3 30 3 3 0.5
+five 4 40 4 4 0.75
+five 5 50 5 5 1.0
+lonely 99 42 1 1 0.0
+six 1 10 1 1 0.0
+six 2 20 2 2 0.2
+six 3 30 3 3 0.4
+six 4 40 4 4 0.6
+six 5 50 5 5 0.8
+six 6 60 6 6 1.0
+PREHOOK: query: select name, rowindex, mynumber,
+rank() over (order by mynumber) as r,
+dense_rank() over (order by mynumber) as dr,
+percent_rank() over (order by mynumber) as pr
+from vector_ptf_percent_rank_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+rank() over (order by mynumber) as r,
+dense_rank() over (order by mynumber) as dr,
+percent_rank() over (order by mynumber) as pr
+from vector_ptf_percent_rank_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+six 1 10 1 1 0.0
+five 1 10 1 1 0.0
+six 2 20 3 2 0.15384615384615385
+five 2 20 3 2 0.15384615384615385
+six 3 30 5 3 0.3076923076923077
+five 3 30 5 3 0.3076923076923077
+five 4 40 7 4 0.46153846153846156
+six 4 40 7 4 0.46153846153846156
+lonely 99 42 9 5 0.6153846153846154
+five 5 50 10 6 0.6923076923076923
+six 5 50 10 6 0.6923076923076923
+six 6 60 12 7 0.8461538461538461
+NULL 1 100 13 8 0.9230769230769231
+NULL 2 100 13 8 0.9230769230769231
+PREHOOK: query: select name, rowindex, mynumber,
+rank() over (partition by name) as r,
+dense_rank() over (partition by name) as dr,
+percent_rank() over (partition by name) as pr
+from vector_ptf_percent_rank_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+rank() over (partition by name) as r,
+dense_rank() over (partition by name) as dr,
+percent_rank() over (partition by name) as pr
+from vector_ptf_percent_rank_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+five 1 10 1 1 0.0
+five 2 20 1 1 0.0
+five 3 30 1 1 0.0
+five 4 40 1 1 0.0
+five 5 50 1 1 0.0
+lonely 99 42 1 1 0.0
+NULL 2 100 1 1 0.0
+NULL 1 100 1 1 0.0
+six 4 40 1 1 0.0
+six 5 50 1 1 0.0
+six 6 60 1 1 0.0
+six 1 10 1 1 0.0
+six 3 30 1 1 0.0
+six 2 20 1 1 0.0
+PREHOOK: query: select name, rowindex, mynumber,
+rank() over () as r,
+dense_rank() over () as dr,
+percent_rank() over () as pr
+from vector_ptf_percent_rank_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+rank() over () as r,
+dense_rank() over () as dr,
+percent_rank() over () as pr
+from vector_ptf_percent_rank_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+five 1 10 1 1 0.0
+five 2 20 1 1 0.0
+five 3 30 1 1 0.0
+five 4 40 1 1 0.0
+five 5 50 1 1 0.0
+six 1 10 1 1 0.0
+six 2 20 1 1 0.0
+six 3 30 1 1 0.0
+six 4 40 1 1 0.0
+six 5 50 1 1 0.0
+six 6 60 1 1 0.0
+lonely 99 42 1 1 0.0
+NULL 1 100 1 1 0.0
+NULL 2 100 1 1 0.0
+PREHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+percent_rank() over (partition by name order by mynumber) as pr
+from vector_ptf_percent_rank_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+percent_rank() over (partition by name order by mynumber) as pr
+from vector_ptf_percent_rank_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: vector_ptf_percent_rank_int
+ Statistics: Num rows: 14 Data size: 1243 Basic stats:
COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:name:string,
1:rowindex:int, 2:mynumber:int,
3:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>,
4:ROW__IS__DELETED:boolean]
+ Reduce Output Operator
+ key expressions: name (type: string), mynumber (type: int)
+ null sort order: az
+ sort order: ++
+ Map-reduce partition columns: name (type: string)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: 0:string, 2:int
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ partitionColumns: 0:string
+ valueColumns: 1:int
+ Statistics: Num rows: 14 Data size: 1243 Basic stats:
COMPLETE Column stats: COMPLETE
+ value expressions: rowindex (type: int)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet:
hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats:
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ includeColumns: [0, 1, 2]
+ dataColumns: name:string, rowindex:int, mynumber:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: az
+ reduceColumnSortOrder: ++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ dataColumns: KEY.reducesinkkey0:string,
KEY.reducesinkkey1:int, VALUE._col0:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [double]
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string), VALUE._col0
(type: int), KEY.reducesinkkey1 (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 2, 1]
+ Statistics: Num rows: 14 Data size: 1243 Basic stats: COMPLETE
Column stats: COMPLETE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string, _col1: int, _col2: int
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col2 ASC NULLS LAST
+ partition by: _col0
+ raw input shape:
+ window functions:
+ window function definition
+ alias: percent_rank_window_0
+ arguments: _col2
+ name: percent_rank
+ window function: GenericUDAFPercentRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorPercentRank]
+ functionInputExpressions: [col 1:int]
+ functionNames: [percent_rank]
+ keyInputColumns: [0, 1]
+ native: true
+ nonKeyInputColumns: [2]
+ orderExpressions: [col 1:int]
+ outputColumns: [3, 0, 2, 1]
+ outputTypes: [double, string, int, int]
+ partitionExpressions: [col 0:string]
+ streamingColumns: [3]
+ Statistics: Num rows: 14 Data size: 1243 Basic stats:
COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: int),
_col2 (type: int), percent_rank_window_0 (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 2, 1, 3]
+ Statistics: Num rows: 14 Data size: 1181 Basic stats:
COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 14 Data size: 1181 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select name, rowindex, mynumber,
+percent_rank() over (partition by name order by mynumber) as pr
+from vector_ptf_percent_rank_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+percent_rank() over (partition by name order by mynumber) as pr
+from vector_ptf_percent_rank_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+NULL 2 100 0.0
+NULL 1 100 0.0
+five 1 10 0.0
+five 2 20 0.25
+five 3 30 0.5
+five 4 40 0.75
+five 5 50 1.0
+lonely 99 42 0.0
+six 1 10 0.0
+six 2 20 0.2
+six 3 30 0.4
+six 4 40 0.6
+six 5 50 0.8
+six 6 60 1.0
+PREHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+rank() over (partition by name order by mynumber) as r,
+dense_rank() over (partition by name order by mynumber) as dr,
+percent_rank() over (partition by name order by mynumber) as pr
+from vector_ptf_percent_rank_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+rank() over (partition by name order by mynumber) as r,
+dense_rank() over (partition by name order by mynumber) as dr,
+percent_rank() over (partition by name order by mynumber) as pr
+from vector_ptf_percent_rank_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: vector_ptf_percent_rank_int
+ Statistics: Num rows: 14 Data size: 1243 Basic stats:
COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:name:string,
1:rowindex:int, 2:mynumber:int,
3:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>,
4:ROW__IS__DELETED:boolean]
+ Reduce Output Operator
+ key expressions: name (type: string), mynumber (type: int)
+ null sort order: az
+ sort order: ++
+ Map-reduce partition columns: name (type: string)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: 0:string, 2:int
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ partitionColumns: 0:string
+ valueColumns: 1:int
+ Statistics: Num rows: 14 Data size: 1243 Basic stats:
COMPLETE Column stats: COMPLETE
+ value expressions: rowindex (type: int)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet:
hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats:
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ includeColumns: [0, 1, 2]
+ dataColumns: name:string, rowindex:int, mynumber:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: az
+ reduceColumnSortOrder: ++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ dataColumns: KEY.reducesinkkey0:string,
KEY.reducesinkkey1:int, VALUE._col0:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, bigint, double]
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string), VALUE._col0
(type: int), KEY.reducesinkkey1 (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 2, 1]
+ Statistics: Num rows: 14 Data size: 1243 Basic stats: COMPLETE
Column stats: COMPLETE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string, _col1: int, _col2: int
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col2 ASC NULLS LAST
+ partition by: _col0
+ raw input shape:
+ window functions:
+ window function definition
+ alias: rank_window_0
+ arguments: _col2
+ name: rank
+ window function: GenericUDAFRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: dense_rank_window_1
+ arguments: _col2
+ name: dense_rank
+ window function: GenericUDAFDenseRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: percent_rank_window_2
+ arguments: _col2
+ name: percent_rank
+ window function: GenericUDAFPercentRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorRank,
VectorPTFEvaluatorDenseRank, VectorPTFEvaluatorPercentRank]
+ functionInputExpressions: [col 1:int, col 1:int, col
1:int]
+ functionNames: [rank, dense_rank, percent_rank]
+ keyInputColumns: [0, 1]
+ native: true
+ nonKeyInputColumns: [2]
+ orderExpressions: [col 1:int]
+ outputColumns: [3, 4, 5, 0, 2, 1]
+ outputTypes: [int, int, double, string, int, int]
+ partitionExpressions: [col 0:string]
+ streamingColumns: [3, 4, 5]
+ Statistics: Num rows: 14 Data size: 1243 Basic stats:
COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: int),
_col2 (type: int), rank_window_0 (type: int), dense_rank_window_1 (type: int),
percent_rank_window_2 (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 2, 1, 3, 4, 5]
+ Statistics: Num rows: 14 Data size: 1293 Basic stats:
COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 14 Data size: 1293 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select name, rowindex, mynumber,
+rank() over (partition by name order by mynumber) as r,
+dense_rank() over (partition by name order by mynumber) as dr,
+percent_rank() over (partition by name order by mynumber) as pr
+from vector_ptf_percent_rank_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+rank() over (partition by name order by mynumber) as r,
+dense_rank() over (partition by name order by mynumber) as dr,
+percent_rank() over (partition by name order by mynumber) as pr
+from vector_ptf_percent_rank_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+NULL 2 100 1 1 0.0
+NULL 1 100 1 1 0.0
+five 1 10 1 1 0.0
+five 2 20 2 2 0.25
+five 3 30 3 3 0.5
+five 4 40 4 4 0.75
+five 5 50 5 5 1.0
+lonely 99 42 1 1 0.0
+six 1 10 1 1 0.0
+six 2 20 2 2 0.2
+six 3 30 3 3 0.4
+six 4 40 4 4 0.6
+six 5 50 5 5 0.8
+six 6 60 6 6 1.0
+PREHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+rank() over (order by mynumber) as r,
+dense_rank() over (order by mynumber) as dr,
+percent_rank() over (order by mynumber) as pr
+from vector_ptf_percent_rank_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+rank() over (order by mynumber) as r,
+dense_rank() over (order by mynumber) as dr,
+percent_rank() over (order by mynumber) as pr
+from vector_ptf_percent_rank_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: vector_ptf_percent_rank_int
+ Statistics: Num rows: 14 Data size: 1243 Basic stats:
COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:name:string,
1:rowindex:int, 2:mynumber:int,
3:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>,
4:ROW__IS__DELETED:boolean]
+ Reduce Output Operator
+ key expressions: 0 (type: int), mynumber (type: int)
+ null sort order: az
+ sort order: ++
+ Map-reduce partition columns: 0 (type: int)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: 5:int, 2:int
+ keyExpressions: ConstantVectorExpression(val 0) ->
5:int
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ partitionColumns: 6:int
+ valueColumns: 0:string, 1:int
+ Statistics: Num rows: 14 Data size: 1243 Basic stats:
COMPLETE Column stats: COMPLETE
+ value expressions: name (type: string), rowindex (type:
int)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet:
hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats:
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ includeColumns: [0, 1, 2]
+ dataColumns: name:string, rowindex:int, mynumber:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, bigint]
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: az
+ reduceColumnSortOrder: ++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 4
+ dataColumns: KEY.reducesinkkey0:int,
KEY.reducesinkkey1:int, VALUE._col0:string, VALUE._col1:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, bigint, double, bigint]
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string), VALUE._col1 (type:
int), KEY.reducesinkkey1 (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [2, 3, 1]
+ Statistics: Num rows: 14 Data size: 1243 Basic stats: COMPLETE
Column stats: COMPLETE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string, _col1: int, _col2: int
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col2 ASC NULLS LAST
+ partition by: 0
+ raw input shape:
+ window functions:
+ window function definition
+ alias: rank_window_0
+ arguments: _col2
+ name: rank
+ window function: GenericUDAFRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: dense_rank_window_1
+ arguments: _col2
+ name: dense_rank
+ window function: GenericUDAFDenseRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: percent_rank_window_2
+ arguments: _col2
+ name: percent_rank
+ window function: GenericUDAFPercentRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorRank,
VectorPTFEvaluatorDenseRank, VectorPTFEvaluatorPercentRank]
+ functionInputExpressions: [col 1:int, col 1:int, col
1:int]
+ functionNames: [rank, dense_rank, percent_rank]
+ keyInputColumns: [1]
+ native: true
+ nonKeyInputColumns: [2, 3]
+ orderExpressions: [col 1:int]
+ outputColumns: [4, 5, 6, 2, 3, 1]
+ outputTypes: [int, int, double, string, int, int]
+ partitionExpressions: [ConstantVectorExpression(val 0)
-> 7:int]
+ streamingColumns: [4, 5, 6]
+ Statistics: Num rows: 14 Data size: 1243 Basic stats:
COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: int),
_col2 (type: int), rank_window_0 (type: int), dense_rank_window_1 (type: int),
percent_rank_window_2 (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [2, 3, 1, 4, 5, 6]
+ Statistics: Num rows: 14 Data size: 1293 Basic stats:
COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 14 Data size: 1293 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select name, rowindex, mynumber,
+rank() over (order by mynumber) as r,
+dense_rank() over (order by mynumber) as dr,
+percent_rank() over (order by mynumber) as pr
+from vector_ptf_percent_rank_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+rank() over (order by mynumber) as r,
+dense_rank() over (order by mynumber) as dr,
+percent_rank() over (order by mynumber) as pr
+from vector_ptf_percent_rank_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+six 1 10 1 1 0.0
+five 1 10 1 1 0.0
+six 2 20 3 2 0.15384615384615385
+five 2 20 3 2 0.15384615384615385
+six 3 30 5 3 0.3076923076923077
+five 3 30 5 3 0.3076923076923077
+five 4 40 7 4 0.46153846153846156
+six 4 40 7 4 0.46153846153846156
+lonely 99 42 9 5 0.6153846153846154
+five 5 50 10 6 0.6923076923076923
+six 5 50 10 6 0.6923076923076923
+six 6 60 12 7 0.8461538461538461
+NULL 1 100 13 8 0.9230769230769231
+NULL 2 100 13 8 0.9230769230769231
+PREHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+rank() over (partition by name) as r,
+dense_rank() over (partition by name) as dr,
+percent_rank() over (partition by name) as pr
+from vector_ptf_percent_rank_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+rank() over (partition by name) as r,
+dense_rank() over (partition by name) as dr,
+percent_rank() over (partition by name) as pr
+from vector_ptf_percent_rank_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: vector_ptf_percent_rank_int
+ Statistics: Num rows: 14 Data size: 1243 Basic stats:
COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:name:string,
1:rowindex:int, 2:mynumber:int,
3:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>,
4:ROW__IS__DELETED:boolean]
+ Reduce Output Operator
+ key expressions: name (type: string)
+ null sort order: a
+ sort order: +
+ Map-reduce partition columns: name (type: string)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkStringOperator
+ keyColumns: 0:string
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 1:int, 2:int
+ Statistics: Num rows: 14 Data size: 1243 Basic stats:
COMPLETE Column stats: COMPLETE
+ value expressions: rowindex (type: int), mynumber (type:
int)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet:
hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats:
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ includeColumns: [0, 1, 2]
+ dataColumns: name:string, rowindex:int, mynumber:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: a
+ reduceColumnSortOrder: +
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ dataColumns: KEY.reducesinkkey0:string, VALUE._col0:int,
VALUE._col1:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, bigint, double]
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string), VALUE._col0
(type: int), VALUE._col1 (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1, 2]
+ Statistics: Num rows: 14 Data size: 1243 Basic stats: COMPLETE
Column stats: COMPLETE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string, _col1: int, _col2: int
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col0 ASC NULLS FIRST
+ partition by: _col0
+ raw input shape:
+ window functions:
+ window function definition
+ alias: rank_window_0
+ arguments: _col0
+ name: rank
+ window function: GenericUDAFRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: dense_rank_window_1
+ arguments: _col0
+ name: dense_rank
+ window function: GenericUDAFDenseRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: percent_rank_window_2
+ arguments: _col0
+ name: percent_rank
+ window function: GenericUDAFPercentRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorRank,
VectorPTFEvaluatorDenseRank, VectorPTFEvaluatorPercentRank]
+ functionInputExpressions: [col 0:string, col 0:string,
col 0:string]
+ functionNames: [rank, dense_rank, percent_rank]
+ keyInputColumns: [0]
+ native: true
+ nonKeyInputColumns: [1, 2]
+ orderExpressions: [col 0:string]
+ outputColumns: [3, 4, 5, 0, 1, 2]
+ outputTypes: [int, int, double, string, int, int]
+ partitionExpressions: [col 0:string]
+ streamingColumns: [3, 4, 5]
+ Statistics: Num rows: 14 Data size: 1243 Basic stats:
COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: int),
_col2 (type: int), rank_window_0 (type: int), dense_rank_window_1 (type: int),
percent_rank_window_2 (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1, 2, 3, 4, 5]
+ Statistics: Num rows: 14 Data size: 1293 Basic stats:
COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 14 Data size: 1293 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select name, rowindex, mynumber,
+rank() over (partition by name) as r,
+dense_rank() over (partition by name) as dr,
+percent_rank() over (partition by name) as pr
+from vector_ptf_percent_rank_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+rank() over (partition by name) as r,
+dense_rank() over (partition by name) as dr,
+percent_rank() over (partition by name) as pr
+from vector_ptf_percent_rank_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+five 1 10 1 1 0.0
+five 2 20 1 1 0.0
+five 3 30 1 1 0.0
+five 4 40 1 1 0.0
+five 5 50 1 1 0.0
+lonely 99 42 1 1 0.0
+NULL 2 100 1 1 0.0
+NULL 1 100 1 1 0.0
+six 4 40 1 1 0.0
+six 5 50 1 1 0.0
+six 6 60 1 1 0.0
+six 1 10 1 1 0.0
+six 3 30 1 1 0.0
+six 2 20 1 1 0.0
+PREHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+rank() over () as r,
+dense_rank() over () as dr,
+percent_rank() over () as pr
+from vector_ptf_percent_rank_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+rank() over () as r,
+dense_rank() over () as dr,
+percent_rank() over () as pr
+from vector_ptf_percent_rank_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: vector_ptf_percent_rank_int
+ Statistics: Num rows: 14 Data size: 1243 Basic stats:
COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:name:string,
1:rowindex:int, 2:mynumber:int,
3:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>,
4:ROW__IS__DELETED:boolean]
+ Reduce Output Operator
+ key expressions: 0 (type: int)
+ null sort order: a
+ sort order: +
+ Map-reduce partition columns: 0 (type: int)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkLongOperator
+ keyColumns: 5:int
+ keyExpressions: ConstantVectorExpression(val 0) ->
5:int
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 0:string, 1:int, 2:int
+ Statistics: Num rows: 14 Data size: 1243 Basic stats:
COMPLETE Column stats: COMPLETE
+ value expressions: name (type: string), rowindex (type:
int), mynumber (type: int)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet:
hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats:
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ includeColumns: [0, 1, 2]
+ dataColumns: name:string, rowindex:int, mynumber:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint]
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: a
+ reduceColumnSortOrder: +
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 4
+ dataColumns: KEY.reducesinkkey0:int, VALUE._col0:string,
VALUE._col1:int, VALUE._col2:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, bigint, double, bigint,
bigint, bigint, bigint, bigint]
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string), VALUE._col1 (type:
int), VALUE._col2 (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [1, 2, 3]
+ Statistics: Num rows: 14 Data size: 1243 Basic stats: COMPLETE
Column stats: COMPLETE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string, _col1: int, _col2: int
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: 0 ASC NULLS FIRST
+ partition by: 0
+ raw input shape:
+ window functions:
+ window function definition
+ alias: rank_window_0
+ arguments: 0
+ name: rank
+ window function: GenericUDAFRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: dense_rank_window_1
+ arguments: 0
+ name: dense_rank
+ window function: GenericUDAFDenseRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: percent_rank_window_2
+ arguments: 0
+ name: percent_rank
+ window function: GenericUDAFPercentRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorRank,
VectorPTFEvaluatorDenseRank, VectorPTFEvaluatorPercentRank]
+ functionInputExpressions: [ConstantVectorExpression(val
0) -> 9:int, ConstantVectorExpression(val 0) -> 10:int,
ConstantVectorExpression(val 0) -> 11:int]
+ functionNames: [rank, dense_rank, percent_rank]
+ keyInputColumns: []
+ native: true
+ nonKeyInputColumns: [1, 2, 3]
+ orderExpressions: [ConstantVectorExpression(val 0) ->
8:int]
+ outputColumns: [4, 5, 6, 1, 2, 3]
+ outputTypes: [int, int, double, string, int, int]
+ partitionExpressions: [ConstantVectorExpression(val 0)
-> 7:int]
+ streamingColumns: [4, 5, 6]
+ Statistics: Num rows: 14 Data size: 1243 Basic stats:
COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: int),
_col2 (type: int), rank_window_0 (type: int), dense_rank_window_1 (type: int),
percent_rank_window_2 (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [1, 2, 3, 4, 5, 6]
+ Statistics: Num rows: 14 Data size: 1293 Basic stats:
COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 14 Data size: 1293 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select name, rowindex, mynumber,
+rank() over () as r,
+dense_rank() over () as dr,
+percent_rank() over () as pr
+from vector_ptf_percent_rank_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+rank() over () as r,
+dense_rank() over () as dr,
+percent_rank() over () as pr
+from vector_ptf_percent_rank_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+five 1 10 1 1 0.0
+five 2 20 1 1 0.0
+five 3 30 1 1 0.0
+five 4 40 1 1 0.0
+five 5 50 1 1 0.0
+six 1 10 1 1 0.0
+six 2 20 1 1 0.0
+six 3 30 1 1 0.0
+six 4 40 1 1 0.0
+six 5 50 1 1 0.0
+six 6 60 1 1 0.0
+lonely 99 42 1 1 0.0
+NULL 1 100 1 1 0.0
+NULL 2 100 1 1 0.0
diff --git
a/ql/src/test/results/clientpositive/llap/vector_ptf_unbuffered_partition_col.q.out
b/ql/src/test/results/clientpositive/llap/vector_ptf_unbuffered_partition_col.q.out
new file mode 100644
index 00000000000..14c2e52efe8
--- /dev/null
+++
b/ql/src/test/results/clientpositive/llap/vector_ptf_unbuffered_partition_col.q.out
@@ -0,0 +1,1059 @@
+PREHOOK: query: explain vectorization detail
+select key, sum(c_float) over(partition by c_float order by c_float) as s
+from cbo_t1 order by key, s
+PREHOOK: type: QUERY
+PREHOOK: Input: default@cbo_t1
+PREHOOK: Input: default@cbo_t1@dt=2014
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization detail
+select key, sum(c_float) over(partition by c_float order by c_float) as s
+from cbo_t1 order by key, s
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@cbo_t1
+POSTHOOK: Input: default@cbo_t1@dt=2014
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: cbo_t1
+ Statistics: Num rows: 20 Data size: 1691 Basic stats:
COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:key:string,
1:value:string, 2:c_int:int, 3:c_float:float, 4:c_boolean:boolean, 5:dt:string,
6:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>,
7:ROW__IS__DELETED:boolean]
+ Reduce Output Operator
+ key expressions: c_float (type: float)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: c_float (type: float)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkMultiKeyOperator
+ keyColumns: 3:float
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 0:string
+ Statistics: Num rows: 20 Data size: 1691 Basic stats:
COMPLETE Column stats: COMPLETE
+ value expressions: key (type: string)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet:
hive.vectorized.use.vector.serde.deserialize IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.hadoop.mapred.TextInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 5
+ includeColumns: [0, 3]
+ dataColumns: key:string, value:string, c_int:int,
c_float:float, c_boolean:boolean
+ partitionColumnCount: 1
+ partitionColumns: dt:string
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: z
+ reduceColumnSortOrder: +
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 2
+ dataColumns: KEY.reducesinkkey0:float, VALUE._col0:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [double]
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string), KEY.reducesinkkey0
(type: float)
+ outputColumnNames: _col0, _col3
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [1, 0]
+ Statistics: Num rows: 20 Data size: 1691 Basic stats: COMPLETE
Column stats: COMPLETE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string, _col3: float
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col3 ASC NULLS LAST
+ partition by: _col3
+ raw input shape:
+ window functions:
+ window function definition
+ alias: sum_window_0
+ arguments: _col3
+ name: sum
+ window function: GenericUDAFSumDouble
+ window frame: RANGE PRECEDING(MAX)~CURRENT
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorDoubleSum]
+ functionInputExpressions: [col 0:float]
+ functionNames: [sum]
+ keyInputColumns: [0]
+ native: true
+ nonKeyInputColumns: [1]
+ orderExpressions: [col 0:float]
+ outputColumns: [2, 1, 0]
+ outputTypes: [double, string, float]
+ partitionExpressions: [col 0:float]
+ streamingColumns: []
+ Statistics: Num rows: 20 Data size: 1691 Basic stats:
COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string), sum_window_0 (type:
double)
+ outputColumnNames: _col0, _col1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [1, 2]
+ Statistics: Num rows: 20 Data size: 1605 Basic stats:
COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type:
double)
+ null sort order: zz
+ sort order: ++
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: 1:string, 2:double
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ Statistics: Num rows: 20 Data size: 1605 Basic stats:
COMPLETE Column stats: COMPLETE
+ Reducer 3
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: zz
+ reduceColumnSortOrder: ++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 2
+ dataColumns: KEY.reducesinkkey0:string,
KEY.reducesinkkey1:double
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string),
KEY.reducesinkkey1 (type: double)
+ outputColumnNames: _col0, _col1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1]
+ Statistics: Num rows: 20 Data size: 1605 Basic stats: COMPLETE
Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 20 Data size: 1605 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select key, sum(c_float) over(partition by c_float order by
c_float) as s
+from cbo_t1 order by key, s
+PREHOOK: type: QUERY
+PREHOOK: Input: default@cbo_t1
+PREHOOK: Input: default@cbo_t1@dt=2014
+#### A masked pattern was here ####
+POSTHOOK: query: select key, sum(c_float) over(partition by c_float order by
c_float) as s
+from cbo_t1 order by key, s
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@cbo_t1
+POSTHOOK: Input: default@cbo_t1@dt=2014
+#### A masked pattern was here ####
+ 1 11.0
+ 1 12.0
+ 1 13.0
+ 1 14.0
+1 1.0
+1 2.0
+1 3.0
+1 4.0
+1 5.0
+1 6.0
+1 7.0
+1 8.0
+1 9.0
+1 10.0
+1 17.0
+1 18.0
+1 15.0
+1 16.0
+NULL NULL
+NULL NULL
+PREHOOK: query: explain vectorization detail
+select key, sum(c_float) over(partition by c_float, key order by c_float,
value) as s
+from cbo_t1 order by key, s
+PREHOOK: type: QUERY
+PREHOOK: Input: default@cbo_t1
+PREHOOK: Input: default@cbo_t1@dt=2014
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization detail
+select key, sum(c_float) over(partition by c_float, key order by c_float,
value) as s
+from cbo_t1 order by key, s
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@cbo_t1
+POSTHOOK: Input: default@cbo_t1@dt=2014
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: cbo_t1
+ Statistics: Num rows: 20 Data size: 3306 Basic stats:
COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:key:string,
1:value:string, 2:c_int:int, 3:c_float:float, 4:c_boolean:boolean, 5:dt:string,
6:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>,
7:ROW__IS__DELETED:boolean]
+ Reduce Output Operator
+ key expressions: c_float (type: float), key (type:
string), value (type: string)
+ null sort order: zaz
+ sort order: +++
+ Map-reduce partition columns: c_float (type: float), key
(type: string)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: 3:float, 0:string, 1:string
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ partitionColumns: 3:float, 0:string
+ Statistics: Num rows: 20 Data size: 3306 Basic stats:
COMPLETE Column stats: COMPLETE
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet:
hive.vectorized.use.vector.serde.deserialize IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.hadoop.mapred.TextInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 5
+ includeColumns: [0, 1, 3]
+ dataColumns: key:string, value:string, c_int:int,
c_float:float, c_boolean:boolean
+ partitionColumnCount: 1
+ partitionColumns: dt:string
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: zaz
+ reduceColumnSortOrder: +++
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ dataColumns: KEY.reducesinkkey0:float,
KEY.reducesinkkey1:string, KEY.reducesinkkey2:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [double]
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey1 (type: string),
KEY.reducesinkkey2 (type: string), KEY.reducesinkkey0 (type: float)
+ outputColumnNames: _col0, _col1, _col3
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [1, 2, 0]
+ Statistics: Num rows: 20 Data size: 3306 Basic stats: COMPLETE
Column stats: COMPLETE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string, _col1: string, _col3:
float
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col3 ASC NULLS LAST, _col1 ASC NULLS LAST
+ partition by: _col3, _col0
+ raw input shape:
+ window functions:
+ window function definition
+ alias: sum_window_0
+ arguments: _col3
+ name: sum
+ window function: GenericUDAFSumDouble
+ window frame: RANGE PRECEDING(MAX)~CURRENT
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorDoubleSum]
+ functionInputExpressions: [col 0:float]
+ functionNames: [sum]
+ keyInputColumns: [1, 2, 0]
+ native: true
+ nonKeyInputColumns: []
+ orderExpressions: [col 0:float, col 2:string]
+ outputColumns: [3, 1, 2, 0]
+ outputTypes: [double, string, string, float]
+ partitionExpressions: [col 0:float, col 1:string]
+ streamingColumns: []
+ Statistics: Num rows: 20 Data size: 3306 Basic stats:
COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string), sum_window_0 (type:
double)
+ outputColumnNames: _col0, _col1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [1, 3]
+ Statistics: Num rows: 20 Data size: 1605 Basic stats:
COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type:
double)
+ null sort order: zz
+ sort order: ++
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: 1:string, 3:double
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ Statistics: Num rows: 20 Data size: 1605 Basic stats:
COMPLETE Column stats: COMPLETE
+ Reducer 3
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: zz
+ reduceColumnSortOrder: ++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 2
+ dataColumns: KEY.reducesinkkey0:string,
KEY.reducesinkkey1:double
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string),
KEY.reducesinkkey1 (type: double)
+ outputColumnNames: _col0, _col1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1]
+ Statistics: Num rows: 20 Data size: 1605 Basic stats: COMPLETE
Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 20 Data size: 1605 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select key, sum(c_float) over(partition by c_float, key order
by c_float, value) as s
+from cbo_t1 order by key, s
+PREHOOK: type: QUERY
+PREHOOK: Input: default@cbo_t1
+PREHOOK: Input: default@cbo_t1@dt=2014
+#### A masked pattern was here ####
+POSTHOOK: query: select key, sum(c_float) over(partition by c_float, key order
by c_float, value) as s
+from cbo_t1 order by key, s
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@cbo_t1
+POSTHOOK: Input: default@cbo_t1@dt=2014
+#### A masked pattern was here ####
+ 1 1.0
+ 1 2.0
+ 1 1.0
+ 1 2.0
+1 1.0
+1 2.0
+1 3.0
+1 4.0
+1 5.0
+1 6.0
+1 7.0
+1 8.0
+1 9.0
+1 10.0
+1 11.0
+1 12.0
+1 1.0
+1 2.0
+NULL NULL
+NULL NULL
+PREHOOK: query: explain vectorization detail
+select key, sum(c_float) over(partition by c_float order by key) as s
+from cbo_t1 order by key, s
+PREHOOK: type: QUERY
+PREHOOK: Input: default@cbo_t1
+PREHOOK: Input: default@cbo_t1@dt=2014
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization detail
+select key, sum(c_float) over(partition by c_float order by key) as s
+from cbo_t1 order by key, s
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@cbo_t1
+POSTHOOK: Input: default@cbo_t1@dt=2014
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: cbo_t1
+ Statistics: Num rows: 20 Data size: 1691 Basic stats:
COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:key:string,
1:value:string, 2:c_int:int, 3:c_float:float, 4:c_boolean:boolean, 5:dt:string,
6:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>,
7:ROW__IS__DELETED:boolean]
+ Reduce Output Operator
+ key expressions: c_float (type: float), key (type: string)
+ null sort order: az
+ sort order: ++
+ Map-reduce partition columns: c_float (type: float)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: 3:float, 0:string
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ partitionColumns: 3:float
+ Statistics: Num rows: 20 Data size: 1691 Basic stats:
COMPLETE Column stats: COMPLETE
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet:
hive.vectorized.use.vector.serde.deserialize IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.hadoop.mapred.TextInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 5
+ includeColumns: [0, 3]
+ dataColumns: key:string, value:string, c_int:int,
c_float:float, c_boolean:boolean
+ partitionColumnCount: 1
+ partitionColumns: dt:string
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ notVectorizedReason: PTF operator: Window function argument
references partition-only column not buffered in vector PTF
+ vectorized: false
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey1 (type: string),
KEY.reducesinkkey0 (type: float)
+ outputColumnNames: _col0, _col3
+ Statistics: Num rows: 20 Data size: 1691 Basic stats: COMPLETE
Column stats: COMPLETE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string, _col3: float
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col0 ASC NULLS LAST
+ partition by: _col3
+ raw input shape:
+ window functions:
+ window function definition
+ alias: sum_window_0
+ arguments: _col3
+ name: sum
+ window function: GenericUDAFSumDouble
+ window frame: RANGE PRECEDING(MAX)~CURRENT
+ Statistics: Num rows: 20 Data size: 1691 Basic stats:
COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string), sum_window_0 (type:
double)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 20 Data size: 1605 Basic stats:
COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type:
double)
+ null sort order: zz
+ sort order: ++
+ Statistics: Num rows: 20 Data size: 1605 Basic stats:
COMPLETE Column stats: COMPLETE
+ Reducer 3
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: zz
+ reduceColumnSortOrder: ++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 2
+ dataColumns: KEY.reducesinkkey0:string,
KEY.reducesinkkey1:double
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string),
KEY.reducesinkkey1 (type: double)
+ outputColumnNames: _col0, _col1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1]
+ Statistics: Num rows: 20 Data size: 1605 Basic stats: COMPLETE
Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 20 Data size: 1605 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select key, sum(c_float) over(partition by c_float order by
key) as s
+from cbo_t1 order by key, s
+PREHOOK: type: QUERY
+PREHOOK: Input: default@cbo_t1
+PREHOOK: Input: default@cbo_t1@dt=2014
+#### A masked pattern was here ####
+POSTHOOK: query: select key, sum(c_float) over(partition by c_float order by
key) as s
+from cbo_t1 order by key, s
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@cbo_t1
+POSTHOOK: Input: default@cbo_t1@dt=2014
+#### A masked pattern was here ####
+ 1 2.0
+ 1 2.0
+ 1 4.0
+ 1 4.0
+1 16.0
+1 16.0
+1 16.0
+1 16.0
+1 16.0
+1 16.0
+1 16.0
+1 16.0
+1 16.0
+1 16.0
+1 16.0
+1 16.0
+1 18.0
+1 18.0
+NULL NULL
+NULL NULL
+PREHOOK: query: explain vectorization detail
+select key, sum(c_float + 1) over(partition by c_float order by key) as s
+from cbo_t1 order by key, s
+PREHOOK: type: QUERY
+PREHOOK: Input: default@cbo_t1
+PREHOOK: Input: default@cbo_t1@dt=2014
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization detail
+select key, sum(c_float + 1) over(partition by c_float order by key) as s
+from cbo_t1 order by key, s
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@cbo_t1
+POSTHOOK: Input: default@cbo_t1@dt=2014
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: cbo_t1
+ Statistics: Num rows: 20 Data size: 1691 Basic stats:
COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:key:string,
1:value:string, 2:c_int:int, 3:c_float:float, 4:c_boolean:boolean, 5:dt:string,
6:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>,
7:ROW__IS__DELETED:boolean]
+ Reduce Output Operator
+ key expressions: c_float (type: float), key (type: string)
+ null sort order: az
+ sort order: ++
+ Map-reduce partition columns: c_float (type: float)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: 3:float, 0:string
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ partitionColumns: 3:float
+ Statistics: Num rows: 20 Data size: 1691 Basic stats:
COMPLETE Column stats: COMPLETE
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet:
hive.vectorized.use.vector.serde.deserialize IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.hadoop.mapred.TextInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 5
+ includeColumns: [0, 3]
+ dataColumns: key:string, value:string, c_int:int,
c_float:float, c_boolean:boolean
+ partitionColumnCount: 1
+ partitionColumns: dt:string
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: az
+ reduceColumnSortOrder: ++
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 2
+ dataColumns: KEY.reducesinkkey0:float,
KEY.reducesinkkey1:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [double, double]
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey1 (type: string),
KEY.reducesinkkey0 (type: float)
+ outputColumnNames: _col0, _col3
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [1, 0]
+ Statistics: Num rows: 20 Data size: 1691 Basic stats: COMPLETE
Column stats: COMPLETE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string, _col3: float
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col0 ASC NULLS LAST
+ partition by: _col3
+ raw input shape:
+ window functions:
+ window function definition
+ alias: sum_window_0
+ arguments: (_col3 + UDFToFloat(1))
+ name: sum
+ window function: GenericUDAFSumDouble
+ window frame: RANGE PRECEDING(MAX)~CURRENT
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorDoubleSum]
+ functionInputExpressions: [DoubleColAddDoubleScalar(col
0:float, val 1.0) -> 3:float]
+ functionNames: [sum]
+ keyInputColumns: [1, 0]
+ native: true
+ nonKeyInputColumns: []
+ orderExpressions: [col 1:string]
+ outputColumns: [2, 1, 0]
+ outputTypes: [double, string, float]
+ partitionExpressions: [col 0:float]
+ streamingColumns: []
+ Statistics: Num rows: 20 Data size: 1691 Basic stats:
COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string), sum_window_0 (type:
double)
+ outputColumnNames: _col0, _col1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [1, 2]
+ Statistics: Num rows: 20 Data size: 1605 Basic stats:
COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type:
double)
+ null sort order: zz
+ sort order: ++
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: 1:string, 2:double
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ Statistics: Num rows: 20 Data size: 1605 Basic stats:
COMPLETE Column stats: COMPLETE
+ Reducer 3
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: zz
+ reduceColumnSortOrder: ++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 2
+ dataColumns: KEY.reducesinkkey0:string,
KEY.reducesinkkey1:double
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string),
KEY.reducesinkkey1 (type: double)
+ outputColumnNames: _col0, _col1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1]
+ Statistics: Num rows: 20 Data size: 1605 Basic stats: COMPLETE
Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 20 Data size: 1605 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select key, sum(c_float + 1) over(partition by c_float order
by key) as s
+from cbo_t1 order by key, s
+PREHOOK: type: QUERY
+PREHOOK: Input: default@cbo_t1
+PREHOOK: Input: default@cbo_t1@dt=2014
+#### A masked pattern was here ####
+POSTHOOK: query: select key, sum(c_float + 1) over(partition by c_float order
by key) as s
+from cbo_t1 order by key, s
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@cbo_t1
+POSTHOOK: Input: default@cbo_t1@dt=2014
+#### A masked pattern was here ####
+ 1 4.0
+ 1 4.0
+ 1 8.0
+ 1 8.0
+1 32.0
+1 32.0
+1 32.0
+1 32.0
+1 32.0
+1 32.0
+1 32.0
+1 32.0
+1 32.0
+1 32.0
+1 32.0
+1 32.0
+1 36.0
+1 36.0
+NULL NULL
+NULL NULL
+PREHOOK: query: explain vectorization detail
+select key, sum(c_float) over(partition by c_float + 1 order by key) as s
+from cbo_t1 order by key, s
+PREHOOK: type: QUERY
+PREHOOK: Input: default@cbo_t1
+PREHOOK: Input: default@cbo_t1@dt=2014
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization detail
+select key, sum(c_float) over(partition by c_float + 1 order by key) as s
+from cbo_t1 order by key, s
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@cbo_t1
+POSTHOOK: Input: default@cbo_t1@dt=2014
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: cbo_t1
+ Statistics: Num rows: 20 Data size: 1691 Basic stats:
COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:key:string,
1:value:string, 2:c_int:int, 3:c_float:float, 4:c_boolean:boolean, 5:dt:string,
6:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>,
7:ROW__IS__DELETED:boolean]
+ Reduce Output Operator
+ key expressions: (c_float + UDFToFloat(1)) (type: float),
key (type: string)
+ null sort order: az
+ sort order: ++
+ Map-reduce partition columns: (c_float + UDFToFloat(1))
(type: float)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: 8:float, 0:string
+ keyExpressions: DoubleColAddDoubleScalar(col 3:float,
val 1.0) -> 8:float
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ partitionColumns: 9:float
+ valueColumns: 3:float
+ Statistics: Num rows: 20 Data size: 1691 Basic stats:
COMPLETE Column stats: COMPLETE
+ value expressions: c_float (type: float)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet:
hive.vectorized.use.vector.serde.deserialize IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.hadoop.mapred.TextInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 5
+ includeColumns: [0, 3]
+ dataColumns: key:string, value:string, c_int:int,
c_float:float, c_boolean:boolean
+ partitionColumnCount: 1
+ partitionColumns: dt:string
+ scratchColumnTypeNames: [double, double]
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: az
+ reduceColumnSortOrder: ++
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ dataColumns: KEY.reducesinkkey0:float,
KEY.reducesinkkey1:string, VALUE._col2:float
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [double, double]
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey1 (type: string), VALUE._col2
(type: float)
+ outputColumnNames: _col0, _col3
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [1, 2]
+ Statistics: Num rows: 20 Data size: 1691 Basic stats: COMPLETE
Column stats: COMPLETE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string, _col3: float
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col0 ASC NULLS LAST
+ partition by: (_col3 + UDFToFloat(1))
+ raw input shape:
+ window functions:
+ window function definition
+ alias: sum_window_0
+ arguments: _col3
+ name: sum
+ window function: GenericUDAFSumDouble
+ window frame: RANGE PRECEDING(MAX)~CURRENT
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorDoubleSum]
+ functionInputExpressions: [col 2:float]
+ functionNames: [sum]
+ keyInputColumns: [1]
+ native: true
+ nonKeyInputColumns: [2]
+ orderExpressions: [col 1:string]
+ outputColumns: [3, 1, 2]
+ outputTypes: [double, string, float]
+ partitionExpressions: [DoubleColAddDoubleScalar(col
2:float, val 1.0) -> 4:float]
+ streamingColumns: []
+ Statistics: Num rows: 20 Data size: 1691 Basic stats:
COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string), sum_window_0 (type:
double)
+ outputColumnNames: _col0, _col1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [1, 3]
+ Statistics: Num rows: 20 Data size: 1605 Basic stats:
COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type:
double)
+ null sort order: zz
+ sort order: ++
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: 1:string, 3:double
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ Statistics: Num rows: 20 Data size: 1605 Basic stats:
COMPLETE Column stats: COMPLETE
+ Reducer 3
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: zz
+ reduceColumnSortOrder: ++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 2
+ dataColumns: KEY.reducesinkkey0:string,
KEY.reducesinkkey1:double
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string),
KEY.reducesinkkey1 (type: double)
+ outputColumnNames: _col0, _col1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1]
+ Statistics: Num rows: 20 Data size: 1605 Basic stats: COMPLETE
Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 20 Data size: 1605 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select key, sum(c_float) over(partition by c_float + 1 order
by key) as s
+from cbo_t1 order by key, s
+PREHOOK: type: QUERY
+PREHOOK: Input: default@cbo_t1
+PREHOOK: Input: default@cbo_t1@dt=2014
+#### A masked pattern was here ####
+POSTHOOK: query: select key, sum(c_float) over(partition by c_float + 1 order
by key) as s
+from cbo_t1 order by key, s
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@cbo_t1
+POSTHOOK: Input: default@cbo_t1@dt=2014
+#### A masked pattern was here ####
+ 1 2.0
+ 1 2.0
+ 1 4.0
+ 1 4.0
+1 16.0
+1 16.0
+1 16.0
+1 16.0
+1 16.0
+1 16.0
+1 16.0
+1 16.0
+1 16.0
+1 16.0
+1 16.0
+1 16.0
+1 18.0
+1 18.0
+NULL NULL
+NULL NULL
diff --git a/ql/src/test/results/clientpositive/llap/vector_windowing.q.out
b/ql/src/test/results/clientpositive/llap/vector_windowing.q.out
index 5ba2c7af513..f0e0c408f60 100644
--- a/ql/src/test/results/clientpositive/llap/vector_windowing.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_windowing.q.out
@@ -3250,7 +3250,7 @@ STAGE PLANS:
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
- notVectorizedReason: PTF operator: percent_rank not in
supported functions [avg, count, cume_dist, dense_rank, first_value, lag,
last_value, lead, max, min, rank, row_number, sum]
+ notVectorizedReason: PTF operator: ntile not in supported
functions [avg, count, cume_dist, dense_rank, first_value, lag, last_value,
lead, max, min, percent_rank, rank, row_number, sum]
vectorized: false
Reduce Operator Tree:
Select Operator
diff --git
a/ql/src/test/results/clientpositive/llap/vector_windowing_gby2.q.out
b/ql/src/test/results/clientpositive/llap/vector_windowing_gby2.q.out
index c2faf5d03a6..f27c31ec9e2 100644
--- a/ql/src/test/results/clientpositive/llap/vector_windowing_gby2.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_windowing_gby2.q.out
@@ -792,16 +792,28 @@ STAGE PLANS:
Statistics: Num rows: 10 Data size: 1005 Basic stats:
COMPLETE Column stats: COMPLETE
value expressions: dense_rank_window_1 (type: int),
_col0 (type: int)
Reducer 5
- Execution mode: llap
+ Execution mode: vectorized, llap
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
- notVectorizedReason: PTF operator: percent_rank not in
supported functions [avg, count, cume_dist, dense_rank, first_value, lag,
last_value, lead, max, min, rank, row_number, sum]
- vectorized: false
+ reduceColumnNullOrder: az
+ reduceColumnSortOrder: ++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 4
+ dataColumns: KEY.reducesinkkey0:int,
KEY.reducesinkkey1:double, VALUE._col0:int, VALUE._col1:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [double]
Reduce Operator Tree:
Select Operator
expressions: VALUE._col0 (type: int), VALUE._col1 (type: int),
KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: double)
outputColumnNames: _col0, _col1, _col6, _col7
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [2, 3, 0, 1]
Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE
Column stats: COMPLETE
PTF Operator
Function definitions:
@@ -823,13 +835,34 @@ STAGE PLANS:
window function: GenericUDAFPercentRankEvaluator
window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
isPivotResult: true
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorPercentRank]
+ functionInputExpressions: [col 1:double]
+ functionNames: [percent_rank]
+ keyInputColumns: [0, 1]
+ native: true
+ nonKeyInputColumns: [2, 3]
+ orderExpressions: [col 1:double]
+ outputColumns: [4, 2, 3, 0, 1]
+ outputTypes: [double, int, int, int, double]
+ partitionExpressions: [col 0:int]
+ streamingColumns: [4]
Statistics: Num rows: 10 Data size: 200 Basic stats:
COMPLETE Column stats: COMPLETE
Select Operator
expressions: _col1 (type: int), _col0 (type: int),
percent_rank_window_2 (type: double)
outputColumnNames: _col0, _col1, _col2
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [3, 2, 4]
Statistics: Num rows: 10 Data size: 160 Basic stats:
COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
Statistics: Num rows: 10 Data size: 160 Basic stats:
COMPLETE Column stats: COMPLETE
table:
input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
diff --git
a/ql/src/test/results/clientpositive/llap/vector_windowing_rank.q.out
b/ql/src/test/results/clientpositive/llap/vector_windowing_rank.q.out
index 45ab7990da0..692661b8946 100644
--- a/ql/src/test/results/clientpositive/llap/vector_windowing_rank.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_windowing_rank.q.out
@@ -904,16 +904,28 @@ STAGE PLANS:
partitionColumnCount: 0
scratchColumnTypeNames: []
Reducer 2
- Execution mode: llap
+ Execution mode: vectorized, llap
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
- notVectorizedReason: PTF operator: percent_rank not in
supported functions [avg, count, cume_dist, dense_rank, first_value, lag,
last_value, lead, max, min, rank, row_number, sum]
- vectorized: false
+ reduceColumnNullOrder: az
+ reduceColumnSortOrder: ++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ dataColumns: KEY.reducesinkkey0:decimal(4,2),
KEY.reducesinkkey1:float, VALUE._col6:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [double]
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey1 (type: float), VALUE._col6
(type: string), KEY.reducesinkkey0 (type: decimal(4,2))
outputColumnNames: _col4, _col7, _col9
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [1, 2, 0]
Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE
Column stats: NONE
PTF Operator
Function definitions:
@@ -935,16 +947,40 @@ STAGE PLANS:
window function: GenericUDAFPercentRankEvaluator
window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
isPivotResult: true
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorPercentRank]
+ functionInputExpressions: [col 1:float]
+ functionNames: [percent_rank]
+ keyInputColumns: [1, 0]
+ native: true
+ nonKeyInputColumns: [2]
+ orderExpressions: [col 1:float]
+ outputColumns: [3, 1, 2, 0]
+ outputTypes: [double, float, string, decimal(4,2)]
+ partitionExpressions: [col 0:decimal(4,2)]
+ streamingColumns: [3]
Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE
Column stats: NONE
Limit
Number of rows: 100
+ Limit Vectorization:
+ className: VectorLimitOperator
+ native: true
Statistics: Num rows: 1 Data size: 300 Basic stats:
COMPLETE Column stats: NONE
Select Operator
expressions: _col7 (type: string), percent_rank_window_0
(type: double)
outputColumnNames: _col0, _col1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [2, 3]
Statistics: Num rows: 1 Data size: 300 Basic stats:
COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
Statistics: Num rows: 1 Data size: 300 Basic stats:
COMPLETE Column stats: NONE
table:
input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
diff --git a/ql/src/test/results/clientpositive/llap/windowing_gby2.q.out
b/ql/src/test/results/clientpositive/llap/windowing_gby2.q.out
index c7a29126299..1155317e66b 100644
--- a/ql/src/test/results/clientpositive/llap/windowing_gby2.q.out
+++ b/ql/src/test/results/clientpositive/llap/windowing_gby2.q.out
@@ -416,7 +416,7 @@ STAGE PLANS:
Statistics: Num rows: 10 Data size: 1005 Basic stats:
COMPLETE Column stats: COMPLETE
value expressions: dense_rank_window_1 (type: int),
_col0 (type: int)
Reducer 5
- Execution mode: llap
+ Execution mode: vectorized, llap
Reduce Operator Tree:
Select Operator
expressions: VALUE._col0 (type: int), VALUE._col1 (type: int),
KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: double)
