This is an automated email from the ASF dual-hosted git repository.
abstractdog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new bff3870c637 HIVE-25124: PTF: Vectorize cume_dist function (#6703)
bff3870c637 is described below
commit bff3870c637184d096c1d6789761e0c0afdd6037
Author: ramitg254 <[email protected]>
AuthorDate: Wed Sep 2 14:43:33 2026 +0530
HIVE-25124: PTF: Vectorize cume_dist function (#6703)
---
.../ql/exec/vector/ptf/VectorPTFEvaluatorBase.java | 48 +-
.../vector/ptf/VectorPTFEvaluatorCumeDist.java | 108 ++
.../vector/ptf/VectorPTFEvaluatorDenseRank.java | 3 +-
.../ql/exec/vector/ptf/VectorPTFEvaluatorRank.java | 3 +-
.../ql/exec/vector/ptf/VectorPTFGroupBatches.java | 55 +-
.../hive/ql/exec/vector/ptf/VectorPTFOperator.java | 11 +-
.../hive/ql/optimizer/physical/Vectorizer.java | 5 +-
.../org/apache/hadoop/hive/ql/plan/PTFDesc.java | 7 +-
.../apache/hadoop/hive/ql/plan/VectorPTFDesc.java | 21 +
.../clientpositive/sketches_rewrite_cume_dist.q | 4 +
.../sketches_rewrite_cume_dist_partition_by.q | 4 +
.../queries/clientpositive/vector_ptf_cume_dist.q | 124 ++
.../sketches_materialized_view_cume_dist.q.out | 2 +-
.../clientpositive/llap/vector_ptf_cume_dist.q.out | 1298 ++++++++++++++++++++
.../clientpositive/llap/vector_windowing.q.out | 88 +-
.../llap/vector_windowing_gby2.q.out | 2 +-
.../llap/vector_windowing_rank.q.out | 44 +-
.../clientpositive/llap/vectorized_ptf.q.out | 2 +-
18 files changed, 1786 insertions(+), 43 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorBase.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorBase.java
index 2f14a8af864..d04c90b275b 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorBase.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorBase.java
@@ -19,6 +19,8 @@
package org.apache.hadoop.hive.ql.exec.vector.ptf;
+import java.util.List;
+
import org.apache.hadoop.hive.ql.exec.vector.ColumnVector.Type;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.expressions.IdentityExpression;
@@ -34,21 +36,33 @@
* on an aggregation's 0 or 1 argument(s) and at some point will fill in an
output column with the
* aggregation result. The aggregation argument is an input column or
expression, or no argument.
*
- * When the aggregation is streaming (e.g. row_number, rank, first_value,
etc), the output column
- * can be filled in immediately by the implementation of evaluateGroupBatch.
+ * <p>Evaluators fall into three categories, distinguished by {@link
#streamsResult()} and
+ * {@link #isGroupAggregatedStreamingEvaluator()}:
+ *
+ * <p><b>Buffered aggregate</b> ({@code streamsResult == false},
+ * {@code isGroupAggregatedStreamingEvaluator == false}): e.g. sum, avg.
+ * The aggregation result is not known until the group batches in the window
range have been
+ * processed via {@link #evaluateGroupBatch}. VectorPTFGroupBatches then uses
+ * {@link #isGroupResultNull()}, {@link #getResultColumnVectorType()}, {@link
#getGroupResult()},
+ * and {@link #getOutputColumnNum()} to fill the output column when group
batches are forwarded.
*
- * For non-streaming aggregations, the aggregation result is not known until
the last group batch
- * is processed. After the last group batch has been processed, the
VectorPTFGroupBatches class
- * will call the isGroupResultNull, getResultColumnVectorType,
getLongGroupResult |
- * getDoubleGroupResult | getDecimalGroupResult, and getOutputColumnNum
methods to get aggregation
- * result information necessary to write it into the output column (as a
repeated column) of all
- * the group batches.
+ * <p><b>Pure streaming</b> ({@code streamsResult == true},
+ * {@code isGroupAggregatedStreamingEvaluator == false}): e.g. row_number,
rank, first_value.
+ * The output column can be filled in by {@link #evaluateGroupBatch} as each
group batch is
+ * forwarded.
+ *
+ * <p><b>Group-aggregated streaming</b> ({@code streamsResult == true},
+ * {@code isGroupAggregatedStreamingEvaluator == true}): e.g. cume_dist.
+ * Group batches are buffered before output is written; {@link
#evaluateGroupBatch} fills the
+ * output column when batches are replayed on forward. See {@link
VectorPTFEvaluatorCumeDist}.
*/
public abstract class VectorPTFEvaluatorBase {
final WindowFrameDef windowFrameDef;
final VectorExpression inputVecExpr;
protected int inputColumnNum;
+ protected int partitionSize = -1;
+
protected final int outputColumnNum;
private boolean nullsLast;
private boolean respectNulls = true;
@@ -198,4 +212,20 @@ public boolean doesRespectNulls() {
public void setRespectNulls(boolean respectNulls) {
this.respectNulls = respectNulls;
}
-}
\ No newline at end of file
+
+ public void setPartitionSize(int partitionSize) {
+ this.partitionSize = partitionSize;
+ }
+
+ public boolean isGroupAggregatedStreamingEvaluator() {
+ return false;
+ }
+
+ /**
+ * Aggregates per peer group result using group row count.
+ */
+ public void addStreamingGroupResults(List<Integer> groupRowCounts) throws
HiveException {
+ throw new HiveException(
+ "No streaming group result precomputation for " +
this.getClass().getName());
+ }
+}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCumeDist.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCumeDist.java
new file mode 100644
index 00000000000..44f4ad8b6ae
--- /dev/null
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCumeDist.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.ptf;
+
+import java.util.ArrayDeque;
+import java.util.Deque;
+import java.util.List;
+
+import org.apache.hadoop.hive.ql.exec.vector.ColumnVector.Type;
+import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.ptf.WindowFrameDef;
+
+/**
+ * Evaluates {@code cume_dist()} as a <b>group-aggregated streaming</b>
evaluator — the third
+ * of three evaluator categories (see {@link VectorPTFEvaluatorBase}).
+ *
+ * <p>It is both <b>buffered</b> and <b>streaming</b>: the operator buffers
the full partition
+ * before any output ({@link #isGroupAggregatedStreamingEvaluator()} {@code ==
true}), then
+ * writes results per batch into column vectors on forward ({@link
#streamsResult()}
+ * {@code == true}). Peer-group values are precomputed in {@link
#addStreamingGroupResults}
+ * after {@link #setPartitionSize(int)}, then {@link #evaluateGroupBatch}
fills the output column
+ * as buffered batches are replayed.
+ */
+public class VectorPTFEvaluatorCumeDist extends VectorPTFEvaluatorBase {
+
+ /**
+ * Per-peer-group cume_dist values from the first pass, popped in order
during
+ * streaming
+ * (one per group at its last batch).
+ */
+ private final Deque<Double> groupResults = new ArrayDeque<>();
+
+ public VectorPTFEvaluatorCumeDist(WindowFrameDef windowFrameDef, int
outputColumnNum) {
+ super(windowFrameDef, outputColumnNum);
+ resetEvaluator();
+ }
+
+ @Override
+ public boolean isGroupAggregatedStreamingEvaluator() {
+ return true;
+ }
+
+ @Override
+ public void addStreamingGroupResults(List<Integer> groupRowCounts) throws
HiveException {
+ if (partitionSize <= 0) {
+ throw new HiveException("Partition size must be set before precomputing
cume_dist");
+ }
+ groupResults.clear();
+ int rowPosition = 0;
+ for (int groupRowCount : groupRowCounts) {
+ rowPosition += groupRowCount;
+ groupResults.addLast((double) rowPosition / partitionSize);
+ }
+ }
+
+ @Override
+ public void evaluateGroupBatch(VectorizedRowBatch batch) throws
HiveException {
+ Double result = groupResults.peekFirst();
+ if (result == null) {
+ throw new HiveException("cume_dist streaming result is not available for
the current group");
+ }
+ DoubleColumnVector outputColVector = (DoubleColumnVector)
batch.cols[outputColumnNum];
+ outputColVector.isRepeating = true;
+ outputColVector.noNulls = true;
+ outputColVector.isNull[0] = false;
+ outputColVector.vector[0] = result;
+ }
+
+ @Override
+ public void doLastBatchWork() {
+ groupResults.pollFirst();
+ }
+
+ @Override
+ public boolean streamsResult() {
+ return true;
+ }
+
+ @Override
+ public Type getResultColumnVectorType() {
+ return Type.DOUBLE;
+ }
+
+ @Override
+ public void resetEvaluator() {
+ partitionSize = -1;
+ groupResults.clear();
+ }
+}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDenseRank.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDenseRank.java
index 6307dd48a5a..58f3f26fd5a 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDenseRank.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDenseRank.java
@@ -46,6 +46,7 @@ public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
// We don't evaluate input columns...
LongColumnVector longColVector = (LongColumnVector)
batch.cols[outputColumnNum];
+ longColVector.noNulls = true;
longColVector.isRepeating = true;
longColVector.isNull[0] = false;
longColVector.vector[0] = denseRank;
@@ -81,4 +82,4 @@ public Object getGroupResult() {
public void resetEvaluator() {
denseRank = 1;
}
-}
\ No newline at end of file
+}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorRank.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorRank.java
index 4055e72b1fa..569b04a26ce 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorRank.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorRank.java
@@ -51,6 +51,7 @@ public void evaluateGroupBatch(VectorizedRowBatch batch)
* Do careful maintenance of the outputColVector.noNulls flag.
*/
LongColumnVector longColVector = (LongColumnVector)
batch.cols[outputColumnNum];
+ longColVector.noNulls = true;
longColVector.isRepeating = true;
longColVector.isNull[0] = false;
longColVector.vector[0] = rank;
@@ -89,4 +90,4 @@ public void resetEvaluator() {
rank = 1;
groupCount = 0;
}
-}
\ No newline at end of file
+}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFGroupBatches.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFGroupBatches.java
index 773c59bda81..41a8413fa8e 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFGroupBatches.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFGroupBatches.java
@@ -94,6 +94,21 @@ public class VectorPTFGroupBatches extends PTFPartition {
List<Integer> inMemoryStartRowIndex;
+ /**
+ * Whether any of the evaluators is a group-aggregated streaming evaluator.
+ */
+ private boolean hasGroupAggregatedStreamingEvaluator;
+
+ /**
+ * List of row counts of each reduce key group (peer group) within the
current partition.
+ */
+ private final List<Integer> aggregatedGroupRowCounts = new ArrayList<>();
+
+ /**
+ * Accumulates the row count of the reduce key group currently being
buffered.
+ */
+ private int currentAggregatedGroupRowCount;
+
/**
* PartitionResults collects results for the currently evaluated partition.
Before HIVE-24761, PTF
* vectorization only took care of unbounded windows, which means that all
of the evaluators run
@@ -316,6 +331,14 @@ public void init(VectorPTFEvaluatorBase[] evaluators,
int[] outputProjectionColu
this.blocks = new VectorSpillBlockContainer(spillLimitBufferedBatchCount,
spillLocalDirs,
bufferedColumnMap, bufferedTypeInfos);
+
+ hasGroupAggregatedStreamingEvaluator = false;
+ for (VectorPTFEvaluatorBase evaluator : evaluators) {
+ if (evaluator.isGroupAggregatedStreamingEvaluator()) {
+ hasGroupAggregatedStreamingEvaluator = true;
+ break;
+ }
+ }
}
/**
@@ -455,6 +478,23 @@ public void finishPartition() throws HiveException {
}
}
+ /**
+ * First pass for peer group aggregated streaming evaluators (e.g. cume_dist)
+ */
+ private void precomputeAggregatedStreamingResults() throws HiveException {
+ if (!hasGroupAggregatedStreamingEvaluator) {
+ return;
+ }
+ List<Integer> groupRowCounts =
+ Collections.unmodifiableList(aggregatedGroupRowCounts);
+ for (VectorPTFEvaluatorBase evaluator : evaluators) {
+ if (evaluator.isGroupAggregatedStreamingEvaluator()) {
+ evaluator.addStreamingGroupResults(groupRowCounts);
+ }
+ }
+ aggregatedGroupRowCounts.clear();
+ }
+
/**
* This should be called, when all of the batches were processed for a group
(all of the
* evaluators have been evaluated for this group). The data is not needed
anymore.
@@ -505,6 +545,14 @@ public void bufferGroupBatch(VectorizedRowBatch batch,
boolean isLastGroupBatch)
copyBufferedColumns(batch, bufferedBatch);
bufferedBatch.isLastGroupBatch = isLastGroupBatch;
cachedSize = -1; // clear cached size as we added new batches
+
+ if (hasGroupAggregatedStreamingEvaluator) {
+ currentAggregatedGroupRowCount += batch.size;
+ if (isLastGroupBatch) {
+ aggregatedGroupRowCounts.add(currentAggregatedGroupRowCount);
+ currentAggregatedGroupRowCount = 0;
+ }
+ }
}
public void resetEvaluators() {
@@ -839,6 +887,11 @@ private void setRepeatingColumn(Object partitionKey,
VectorizedRowBatch targetBa
void preFinishPartition() throws HiveException {
int rows = size();
+ for (VectorPTFEvaluatorBase evaluator : evaluators) {
+ evaluator.setPartitionSize(rows);
+ }
+ precomputeAggregatedStreamingResults();
+
positionCache = new RowPositionInBatch[rows + 1];
partitionResults = new PartitionResults(rows);
jumpToFirstBlock();
@@ -1215,4 +1268,4 @@ private void copyBufferedColumns(VectorizedRowBatch
batch, VectorizedRowBatch bu
bufferedBatch.size = size;
}
-}
\ No newline at end of file
+}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFOperator.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFOperator.java
index 11dd51f9dbf..38e9a6d911f 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFOperator.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFOperator.java
@@ -103,9 +103,6 @@ public class VectorPTFOperator extends Operator<PTFDesc>
private TypeInfo[] outputTypeInfos;
private DataTypePhysicalVariation[] outputDataTypePhysicalVariations;
- private int evaluatorCount;
- private String[] evaluatorFunctionNames;
-
private int[] orderColumnMap;
private Type[] orderColumnVectorTypes;
private VectorExpression[] orderExpressions;
@@ -192,9 +189,6 @@ public VectorPTFOperator(CompilationOpContext ctx,
OperatorDesc conf,
vOutContext = new VectorizationContext(getName(), this.vContext);
setupVOutContext();
- evaluatorFunctionNames = this.vectorDesc.getEvaluatorFunctionNames();
- evaluatorCount = evaluatorFunctionNames.length;
-
orderColumnMap = vectorPTFInfo.getOrderColumnMap();
orderColumnVectorTypes = vectorPTFInfo.getOrderColumnVectorTypes();
orderExpressions = vectorPTFInfo.getOrderExpressions();
@@ -333,8 +327,7 @@ protected void initializeOp(Configuration hconf) throws
HiveException {
}
streamingEvaluatorNums =
VectorPTFDesc.getStreamingEvaluatorNums(evaluators);
-
- allEvaluatorsAreStreaming = (streamingEvaluatorNums.length ==
evaluatorCount);
+ allEvaluatorsAreStreaming =
VectorPTFDesc.getAllEvaluatorsAreStreaming(evaluators);
groupBatches = new VectorPTFGroupBatches(
hconf, vectorDesc.getVectorizedPTFMaxMemoryBufferingBatchCount());
@@ -822,4 +815,4 @@ public VectorizationContext getInputVectorizationContext() {
public VectorDesc getVectorDesc() {
return vectorDesc;
}
-}
\ No newline at end of file
+}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
index 12c43f7f6e8..00cdcc50ad1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
@@ -3005,9 +3005,10 @@ private boolean validatePTFOperator(PTFOperator op,
VectorizationContext vContex
throw new RuntimeException("Unexpected window type " +
windowFrameDef.getWindowType());
}
- // RANK/DENSE_RANK don't care about columns.
+ // RANK/DENSE_RANK/CUME_DIST don't care about columns.
if (supportedFunctionType != SupportedFunctionType.RANK &&
- supportedFunctionType != SupportedFunctionType.DENSE_RANK) {
+ supportedFunctionType != SupportedFunctionType.DENSE_RANK &&
+ supportedFunctionType != SupportedFunctionType.CUME_DIST) {
if (exprNodeDescList != null) {
// LEAD and LAG now supports multiple arguments in vectorized mode
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java
b/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java
index 01e3f4f6708..c0b73821538 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java
@@ -226,12 +226,7 @@ public String getStreamingColumns() {
public boolean getAllEvaluatorsAreStreaming() {
VectorPTFEvaluatorBase[] evaluators =
VectorPTFDesc.getEvaluators(vectorPTFDesc, vectorPTFInfo);
- for (VectorPTFEvaluatorBase evaluator : evaluators) {
- if (!evaluator.streamsResult()) {
- return false;
- }
- }
- return true;
+ return VectorPTFDesc.getAllEvaluatorsAreStreaming(evaluators);
}
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorPTFDesc.java
b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorPTFDesc.java
index b19fdddb418..aab04bee56c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorPTFDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorPTFDesc.java
@@ -40,6 +40,7 @@
import org.apache.hadoop.hive.ql.exec.vector.ptf.VectorPTFEvaluatorDecimalMax;
import org.apache.hadoop.hive.ql.exec.vector.ptf.VectorPTFEvaluatorDecimalMin;
import org.apache.hadoop.hive.ql.exec.vector.ptf.VectorPTFEvaluatorDecimalSum;
+import org.apache.hadoop.hive.ql.exec.vector.ptf.VectorPTFEvaluatorCumeDist;
import org.apache.hadoop.hive.ql.exec.vector.ptf.VectorPTFEvaluatorDenseRank;
import org.apache.hadoop.hive.ql.exec.vector.ptf.VectorPTFEvaluatorDoubleAvg;
import
org.apache.hadoop.hive.ql.exec.vector.ptf.VectorPTFEvaluatorDoubleCountDistinct;
@@ -92,6 +93,7 @@ public enum SupportedFunctionType {
ROW_NUMBER,
RANK,
DENSE_RANK,
+ CUME_DIST,
MIN,
MAX,
SUM,
@@ -202,6 +204,9 @@ public static VectorPTFEvaluatorBase
getEvaluator(SupportedFunctionType function
case DENSE_RANK:
evaluator = new VectorPTFEvaluatorDenseRank(windowFrameDef,
outputColumnNum);
break;
+ case CUME_DIST:
+ evaluator = new VectorPTFEvaluatorCumeDist(windowFrameDef,
outputColumnNum);
+ break;
case MIN:
switch (columnVectorType) {
case LONG:
@@ -468,6 +473,22 @@ public static int[]
getStreamingEvaluatorNums(VectorPTFEvaluatorBase[] evaluator
return ArrayUtils.toPrimitive(streamingEvaluatorNums.toArray(new
Integer[0]));
}
+ /**
+ * Returns whether every evaluator can process the batch immediately via
+ * {@link org.apache.hadoop.hive.ql.exec.vector.ptf.VectorPTFGroupBatches
+ * #evaluateStreamingGroupBatch} without buffering the partition.
+ *
+ * <p>See {@link VectorPTFEvaluatorBase} for evaluator categories and flag
semantics.
+ */
+ public static boolean getAllEvaluatorsAreStreaming(VectorPTFEvaluatorBase[]
evaluators) {
+ for (VectorPTFEvaluatorBase evaluator : evaluators) {
+ if (!evaluator.streamsResult() ||
evaluator.isGroupAggregatedStreamingEvaluator()) {
+ return false;
+ }
+ }
+ return true;
+ }
+
public TypeInfo[] getReducerBatchTypeInfos() {
return reducerBatchTypeInfos;
}
diff --git a/ql/src/test/queries/clientpositive/sketches_rewrite_cume_dist.q
b/ql/src/test/queries/clientpositive/sketches_rewrite_cume_dist.q
index 33e0f6528a2..a7ef8cb2285 100644
--- a/ql/src/test/queries/clientpositive/sketches_rewrite_cume_dist.q
+++ b/ql/src/test/queries/clientpositive/sketches_rewrite_cume_dist.q
@@ -12,11 +12,15 @@ insert into table sketch_input values
select id,cume_dist() over (order by id) from sketch_input;
+-- Vectorized PTF buffered path does not support complex-type passthrough
columns.
+set hive.vectorized.execution.ptf.enabled=false;
+
select id,cume_dist() over (order by id),ds_kll_cdf(ds, CAST(id AS FLOAT) )[0]
from sketch_input
join ( select ds_kll_sketch(cast(id as float)) as ds from sketch_input ) q
order by id;
+set hive.vectorized.execution.ptf.enabled=true;
set hive.optimize.bi.enabled=true;
-- see if rewrite happens
diff --git
a/ql/src/test/queries/clientpositive/sketches_rewrite_cume_dist_partition_by.q
b/ql/src/test/queries/clientpositive/sketches_rewrite_cume_dist_partition_by.q
index 6622bb3b8b8..2449c679ea2 100644
---
a/ql/src/test/queries/clientpositive/sketches_rewrite_cume_dist_partition_by.q
+++
b/ql/src/test/queries/clientpositive/sketches_rewrite_cume_dist_partition_by.q
@@ -10,11 +10,15 @@ insert into table sketch_input values
(1,null),(2,null),(10,null),(13,null);
;
+-- Vectorized PTF buffered path does not support complex-type passthrough
columns.
+set hive.vectorized.execution.ptf.enabled=false;
+
select id,category,cume_dist() over (partition by category order by
id),ds_kll_cdf(ds, CAST(id AS FLOAT))[0]
from sketch_input
join ( select category as c,ds_kll_sketch(cast(id as float)) as ds from
sketch_input group by category) q on (q.c=category)
order by category,id;
+set hive.vectorized.execution.ptf.enabled=true;
set hive.optimize.bi.enabled=true;
-- see if rewrite happens
diff --git a/ql/src/test/queries/clientpositive/vector_ptf_cume_dist.q
b/ql/src/test/queries/clientpositive/vector_ptf_cume_dist.q
new file mode 100644
index 00000000000..e8df4d0d485
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/vector_ptf_cume_dist.q
@@ -0,0 +1,124 @@
+set hive.vectorized.testing.reducer.batch.size=2;
+
+DROP TABLE IF EXISTS vector_ptf_cume_dist_int;
+
+CREATE TABLE vector_ptf_cume_dist_int(name string, rowindex int, mynumber int)
stored as orc;
+
+INSERT INTO vector_ptf_cume_dist_int values
+-- first partition (12 rows): cume_dist peer-group values 0.25, 1/3, 0.5, 2/3,
5/6, 1.0
+('first', 1, 1),
+('first', 2, 1),
+('first', 3, 1),
+('first', 4, 2),
+('first', 5, 3),
+('first', 6, 3),
+('first', 7, 4),
+('first', 8, 4),
+('first', 9, 5),
+('first', 10, 5),
+('first', 11, NULL),
+('first', 12, NULL),
+-- second partition (10 rows): cume_dist peer-group values 0.2, 0.4, 0.5, 0.7,
0.8, 1.0
+('second', 22, 10),
+('second', 23, 10),
+('second', 24, 20),
+('second', 25, 20),
+('second', 26, 30),
+('second', 27, 40),
+('second', 28, 40),
+('second', 29, 50),
+('second', 30, NULL),
+('second', 31, NULL),
+-- null partition
+(NULL, 43, 7),
+(NULL, 44, 7);
+
+-- NON-VECTORIZED
+set hive.vectorized.execution.ptf.enabled=false;
+
+select name, rowindex, mynumber,
+cume_dist() over (partition by name order by mynumber) as cud
+from vector_ptf_cume_dist_int;
+
+select name, rowindex, mynumber,
+rank() over (partition by name order by mynumber) as r,
+dense_rank() over (partition by name order by mynumber) as dr,
+cume_dist() over (partition by name order by mynumber) as cud
+from vector_ptf_cume_dist_int;
+
+select name, rowindex, mynumber,
+rank() over (order by mynumber) as r,
+dense_rank() over (order by mynumber) as dr,
+cume_dist() over (order by mynumber) as cud
+from vector_ptf_cume_dist_int;
+
+select name, rowindex, mynumber,
+rank() over (partition by name) as r,
+dense_rank() over (partition by name) as dr,
+cume_dist() over (partition by name) as cud
+from vector_ptf_cume_dist_int;
+
+select name, rowindex, mynumber,
+rank() over () as r,
+dense_rank() over () as dr,
+cume_dist() over () as cud
+from vector_ptf_cume_dist_int;
+
+-- VECTORIZED
+set hive.vectorized.execution.ptf.enabled=true;
+
+explain vectorization detail select name, rowindex, mynumber,
+cume_dist() over (partition by name order by mynumber) as cud
+from vector_ptf_cume_dist_int;
+
+select name, rowindex, mynumber,
+cume_dist() over (partition by name order by mynumber) as cud
+from vector_ptf_cume_dist_int;
+
+explain vectorization detail select name, rowindex, mynumber,
+rank() over (partition by name order by mynumber) as r,
+dense_rank() over (partition by name order by mynumber) as dr,
+cume_dist() over (partition by name order by mynumber) as cud
+from vector_ptf_cume_dist_int;
+
+select name, rowindex, mynumber,
+rank() over (partition by name order by mynumber) as r,
+dense_rank() over (partition by name order by mynumber) as dr,
+cume_dist() over (partition by name order by mynumber) as cud
+from vector_ptf_cume_dist_int;
+
+explain vectorization detail select name, rowindex, mynumber,
+rank() over (order by mynumber) as r,
+dense_rank() over (order by mynumber) as dr,
+cume_dist() over (order by mynumber) as cud
+from vector_ptf_cume_dist_int;
+
+select name, rowindex, mynumber,
+rank() over (order by mynumber) as r,
+dense_rank() over (order by mynumber) as dr,
+cume_dist() over (order by mynumber) as cud
+from vector_ptf_cume_dist_int;
+
+explain vectorization detail select name, rowindex, mynumber,
+rank() over (partition by name) as r,
+dense_rank() over (partition by name) as dr,
+cume_dist() over (partition by name) as cud
+from vector_ptf_cume_dist_int;
+
+select name, rowindex, mynumber,
+rank() over (partition by name) as r,
+dense_rank() over (partition by name) as dr,
+cume_dist() over (partition by name) as cud
+from vector_ptf_cume_dist_int;
+
+explain vectorization detail select name, rowindex, mynumber,
+rank() over () as r,
+dense_rank() over () as dr,
+cume_dist() over () as cud
+from vector_ptf_cume_dist_int;
+
+select name, rowindex, mynumber,
+rank() over () as r,
+dense_rank() over () as dr,
+cume_dist() over () as cud
+from vector_ptf_cume_dist_int;
diff --git
a/ql/src/test/results/clientpositive/llap/sketches_materialized_view_cume_dist.q.out
b/ql/src/test/results/clientpositive/llap/sketches_materialized_view_cume_dist.q.out
index b09f1fbfb2a..96f4236c95e 100644
---
a/ql/src/test/results/clientpositive/llap/sketches_materialized_view_cume_dist.q.out
+++
b/ql/src/test/results/clientpositive/llap/sketches_materialized_view_cume_dist.q.out
@@ -236,7 +236,7 @@ STAGE PLANS:
Execution mode: vectorized, llap
LLAP IO: may be used (ACID table)
Reducer 2
- Execution mode: llap
+ Execution mode: vectorized, llap
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey1 (type: int)
diff --git a/ql/src/test/results/clientpositive/llap/vector_ptf_cume_dist.q.out
b/ql/src/test/results/clientpositive/llap/vector_ptf_cume_dist.q.out
new file mode 100644
index 00000000000..b0532820422
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/vector_ptf_cume_dist.q.out
@@ -0,0 +1,1298 @@
+PREHOOK: query: DROP TABLE IF EXISTS vector_ptf_cume_dist_int
+PREHOOK: type: DROPTABLE
+PREHOOK: Output: database:default
+POSTHOOK: query: DROP TABLE IF EXISTS vector_ptf_cume_dist_int
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: database:default
+PREHOOK: query: CREATE TABLE vector_ptf_cume_dist_int(name string, rowindex
int, mynumber int) stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@vector_ptf_cume_dist_int
+POSTHOOK: query: CREATE TABLE vector_ptf_cume_dist_int(name string, rowindex
int, mynumber int) stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@vector_ptf_cume_dist_int
+PREHOOK: query: INSERT INTO vector_ptf_cume_dist_int values
+
+('first', 1, 1),
+('first', 2, 1),
+('first', 3, 1),
+('first', 4, 2),
+('first', 5, 3),
+('first', 6, 3),
+('first', 7, 4),
+('first', 8, 4),
+('first', 9, 5),
+('first', 10, 5),
+('first', 11, NULL),
+('first', 12, NULL),
+
+('second', 22, 10),
+('second', 23, 10),
+('second', 24, 20),
+('second', 25, 20),
+('second', 26, 30),
+('second', 27, 40),
+('second', 28, 40),
+('second', 29, 50),
+('second', 30, NULL),
+('second', 31, NULL),
+
+(NULL, 43, 7),
+(NULL, 44, 7)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@vector_ptf_cume_dist_int
+POSTHOOK: query: INSERT INTO vector_ptf_cume_dist_int values
+
+('first', 1, 1),
+('first', 2, 1),
+('first', 3, 1),
+('first', 4, 2),
+('first', 5, 3),
+('first', 6, 3),
+('first', 7, 4),
+('first', 8, 4),
+('first', 9, 5),
+('first', 10, 5),
+('first', 11, NULL),
+('first', 12, NULL),
+
+('second', 22, 10),
+('second', 23, 10),
+('second', 24, 20),
+('second', 25, 20),
+('second', 26, 30),
+('second', 27, 40),
+('second', 28, 40),
+('second', 29, 50),
+('second', 30, NULL),
+('second', 31, NULL),
+
+(NULL, 43, 7),
+(NULL, 44, 7)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@vector_ptf_cume_dist_int
+POSTHOOK: Lineage: vector_ptf_cume_dist_int.mynumber SCRIPT []
+POSTHOOK: Lineage: vector_ptf_cume_dist_int.name SCRIPT []
+POSTHOOK: Lineage: vector_ptf_cume_dist_int.rowindex SCRIPT []
+PREHOOK: query: select name, rowindex, mynumber,
+cume_dist() over (partition by name order by mynumber) as cud
+from vector_ptf_cume_dist_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_cume_dist_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+cume_dist() over (partition by name order by mynumber) as cud
+from vector_ptf_cume_dist_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_cume_dist_int
+#### A masked pattern was here ####
+NULL 44 7 1.0
+NULL 43 7 1.0
+first 1 1 0.25
+first 2 1 0.25
+first 3 1 0.25
+first 4 2 0.3333333333333333
+first 6 3 0.5
+first 5 3 0.5
+first 8 4 0.6666666666666666
+first 7 4 0.6666666666666666
+first 9 5 0.8333333333333334
+first 10 5 0.8333333333333334
+first 11 NULL 1.0
+first 12 NULL 1.0
+second 22 10 0.2
+second 23 10 0.2
+second 25 20 0.4
+second 24 20 0.4
+second 26 30 0.5
+second 27 40 0.7
+second 28 40 0.7
+second 29 50 0.8
+second 30 NULL 1.0
+second 31 NULL 1.0
+PREHOOK: query: select name, rowindex, mynumber,
+rank() over (partition by name order by mynumber) as r,
+dense_rank() over (partition by name order by mynumber) as dr,
+cume_dist() over (partition by name order by mynumber) as cud
+from vector_ptf_cume_dist_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_cume_dist_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+rank() over (partition by name order by mynumber) as r,
+dense_rank() over (partition by name order by mynumber) as dr,
+cume_dist() over (partition by name order by mynumber) as cud
+from vector_ptf_cume_dist_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_cume_dist_int
+#### A masked pattern was here ####
+NULL 44 7 1 1 1.0
+NULL 43 7 1 1 1.0
+first 1 1 1 1 0.25
+first 2 1 1 1 0.25
+first 3 1 1 1 0.25
+first 4 2 4 2 0.3333333333333333
+first 6 3 5 3 0.5
+first 5 3 5 3 0.5
+first 8 4 7 4 0.6666666666666666
+first 7 4 7 4 0.6666666666666666
+first 9 5 9 5 0.8333333333333334
+first 10 5 9 5 0.8333333333333334
+first 11 NULL 11 6 1.0
+first 12 NULL 11 6 1.0
+second 22 10 1 1 0.2
+second 23 10 1 1 0.2
+second 25 20 3 2 0.4
+second 24 20 3 2 0.4
+second 26 30 5 3 0.5
+second 27 40 6 4 0.7
+second 28 40 6 4 0.7
+second 29 50 8 5 0.8
+second 30 NULL 9 6 1.0
+second 31 NULL 9 6 1.0
+PREHOOK: query: select name, rowindex, mynumber,
+rank() over (order by mynumber) as r,
+dense_rank() over (order by mynumber) as dr,
+cume_dist() over (order by mynumber) as cud
+from vector_ptf_cume_dist_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_cume_dist_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+rank() over (order by mynumber) as r,
+dense_rank() over (order by mynumber) as dr,
+cume_dist() over (order by mynumber) as cud
+from vector_ptf_cume_dist_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_cume_dist_int
+#### A masked pattern was here ####
+first 1 1 1 1 0.125
+first 2 1 1 1 0.125
+first 3 1 1 1 0.125
+first 4 2 4 2 0.16666666666666666
+first 5 3 5 3 0.25
+first 6 3 5 3 0.25
+first 7 4 7 4 0.3333333333333333
+first 8 4 7 4 0.3333333333333333
+first 9 5 9 5 0.4166666666666667
+first 10 5 9 5 0.4166666666666667
+NULL 44 7 11 6 0.5
+NULL 43 7 11 6 0.5
+second 23 10 13 7 0.5833333333333334
+second 22 10 13 7 0.5833333333333334
+second 24 20 15 8 0.6666666666666666
+second 25 20 15 8 0.6666666666666666
+second 26 30 17 9 0.7083333333333334
+second 27 40 18 10 0.7916666666666666
+second 28 40 18 10 0.7916666666666666
+second 29 50 20 11 0.8333333333333334
+first 11 NULL 21 12 1.0
+second 30 NULL 21 12 1.0
+second 31 NULL 21 12 1.0
+first 12 NULL 21 12 1.0
+PREHOOK: query: select name, rowindex, mynumber,
+rank() over (partition by name) as r,
+dense_rank() over (partition by name) as dr,
+cume_dist() over (partition by name) as cud
+from vector_ptf_cume_dist_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_cume_dist_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+rank() over (partition by name) as r,
+dense_rank() over (partition by name) as dr,
+cume_dist() over (partition by name) as cud
+from vector_ptf_cume_dist_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_cume_dist_int
+#### A masked pattern was here ####
+NULL 44 7 1 1 1.0
+NULL 43 7 1 1 1.0
+first 1 1 1 1 1.0
+first 2 1 1 1 1.0
+first 3 1 1 1 1.0
+first 4 2 1 1 1.0
+first 5 3 1 1 1.0
+first 6 3 1 1 1.0
+first 7 4 1 1 1.0
+first 8 4 1 1 1.0
+first 9 5 1 1 1.0
+first 10 5 1 1 1.0
+first 11 NULL 1 1 1.0
+first 12 NULL 1 1 1.0
+second 24 20 1 1 1.0
+second 25 20 1 1 1.0
+second 26 30 1 1 1.0
+second 27 40 1 1 1.0
+second 28 40 1 1 1.0
+second 29 50 1 1 1.0
+second 30 NULL 1 1 1.0
+second 31 NULL 1 1 1.0
+second 23 10 1 1 1.0
+second 22 10 1 1 1.0
+PREHOOK: query: select name, rowindex, mynumber,
+rank() over () as r,
+dense_rank() over () as dr,
+cume_dist() over () as cud
+from vector_ptf_cume_dist_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_cume_dist_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+rank() over () as r,
+dense_rank() over () as dr,
+cume_dist() over () as cud
+from vector_ptf_cume_dist_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_cume_dist_int
+#### A masked pattern was here ####
+first 1 1 1 1 1.0
+first 2 1 1 1 1.0
+first 3 1 1 1 1.0
+first 4 2 1 1 1.0
+first 5 3 1 1 1.0
+first 6 3 1 1 1.0
+first 7 4 1 1 1.0
+first 8 4 1 1 1.0
+first 9 5 1 1 1.0
+first 10 5 1 1 1.0
+first 11 NULL 1 1 1.0
+first 12 NULL 1 1 1.0
+second 22 10 1 1 1.0
+second 23 10 1 1 1.0
+second 24 20 1 1 1.0
+second 25 20 1 1 1.0
+second 26 30 1 1 1.0
+second 27 40 1 1 1.0
+second 28 40 1 1 1.0
+second 29 50 1 1 1.0
+second 30 NULL 1 1 1.0
+second 31 NULL 1 1 1.0
+NULL 43 7 1 1 1.0
+NULL 44 7 1 1 1.0
+PREHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+cume_dist() over (partition by name order by mynumber) as cud
+from vector_ptf_cume_dist_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_cume_dist_int
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+cume_dist() over (partition by name order by mynumber) as cud
+from vector_ptf_cume_dist_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_cume_dist_int
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: vector_ptf_cume_dist_int
+ Statistics: Num rows: 24 Data size: 2227 Basic stats:
COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:name:string,
1:rowindex:int, 2:mynumber:int,
3:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>,
4:ROW__IS__DELETED:boolean]
+ Reduce Output Operator
+ key expressions: name (type: string), mynumber (type: int)
+ null sort order: az
+ sort order: ++
+ Map-reduce partition columns: name (type: string)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: 0:string, 2:int
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ partitionColumns: 0:string
+ valueColumns: 1:int
+ Statistics: Num rows: 24 Data size: 2227 Basic stats:
COMPLETE Column stats: COMPLETE
+ value expressions: rowindex (type: int)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet:
hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats:
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ includeColumns: [0, 1, 2]
+ dataColumns: name:string, rowindex:int, mynumber:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: az
+ reduceColumnSortOrder: ++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ dataColumns: KEY.reducesinkkey0:string,
KEY.reducesinkkey1:int, VALUE._col0:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [double]
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string), VALUE._col0
(type: int), KEY.reducesinkkey1 (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 2, 1]
+ Statistics: Num rows: 24 Data size: 2227 Basic stats: COMPLETE
Column stats: COMPLETE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string, _col1: int, _col2: int
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col2 ASC NULLS LAST
+ partition by: _col0
+ raw input shape:
+ window functions:
+ window function definition
+ alias: cume_dist_window_0
+ arguments: _col2
+ name: cume_dist
+ window function: GenericUDAFCumeDistEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorCumeDist]
+ functionInputExpressions: [col 1:int]
+ functionNames: [cume_dist]
+ keyInputColumns: [0, 1]
+ native: true
+ nonKeyInputColumns: [2]
+ orderExpressions: [col 1:int]
+ outputColumns: [3, 0, 2, 1]
+ outputTypes: [double, string, int, int]
+ partitionExpressions: [col 0:string]
+ streamingColumns: [3]
+ Statistics: Num rows: 24 Data size: 2227 Basic stats:
COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: int),
_col2 (type: int), cume_dist_window_0 (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 2, 1, 3]
+ Statistics: Num rows: 24 Data size: 2225 Basic stats:
COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 24 Data size: 2225 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select name, rowindex, mynumber,
+cume_dist() over (partition by name order by mynumber) as cud
+from vector_ptf_cume_dist_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_cume_dist_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+cume_dist() over (partition by name order by mynumber) as cud
+from vector_ptf_cume_dist_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_cume_dist_int
+#### A masked pattern was here ####
+NULL 44 7 1.0
+NULL 43 7 1.0
+first 1 1 0.25
+first 2 1 0.25
+first 3 1 0.25
+first 4 2 0.3333333333333333
+first 6 3 0.5
+first 5 3 0.5
+first 8 4 0.6666666666666666
+first 7 4 0.6666666666666666
+first 9 5 0.8333333333333334
+first 10 5 0.8333333333333334
+first 11 NULL 1.0
+first 12 NULL 1.0
+second 22 10 0.2
+second 23 10 0.2
+second 25 20 0.4
+second 24 20 0.4
+second 26 30 0.5
+second 27 40 0.7
+second 28 40 0.7
+second 29 50 0.8
+second 30 NULL 1.0
+second 31 NULL 1.0
+PREHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+rank() over (partition by name order by mynumber) as r,
+dense_rank() over (partition by name order by mynumber) as dr,
+cume_dist() over (partition by name order by mynumber) as cud
+from vector_ptf_cume_dist_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_cume_dist_int
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+rank() over (partition by name order by mynumber) as r,
+dense_rank() over (partition by name order by mynumber) as dr,
+cume_dist() over (partition by name order by mynumber) as cud
+from vector_ptf_cume_dist_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_cume_dist_int
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: vector_ptf_cume_dist_int
+ Statistics: Num rows: 24 Data size: 2227 Basic stats:
COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:name:string,
1:rowindex:int, 2:mynumber:int,
3:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>,
4:ROW__IS__DELETED:boolean]
+ Reduce Output Operator
+ key expressions: name (type: string), mynumber (type: int)
+ null sort order: az
+ sort order: ++
+ Map-reduce partition columns: name (type: string)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: 0:string, 2:int
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ partitionColumns: 0:string
+ valueColumns: 1:int
+ Statistics: Num rows: 24 Data size: 2227 Basic stats:
COMPLETE Column stats: COMPLETE
+ value expressions: rowindex (type: int)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet:
hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats:
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ includeColumns: [0, 1, 2]
+ dataColumns: name:string, rowindex:int, mynumber:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: az
+ reduceColumnSortOrder: ++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ dataColumns: KEY.reducesinkkey0:string,
KEY.reducesinkkey1:int, VALUE._col0:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, bigint, double]
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string), VALUE._col0
(type: int), KEY.reducesinkkey1 (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 2, 1]
+ Statistics: Num rows: 24 Data size: 2227 Basic stats: COMPLETE
Column stats: COMPLETE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string, _col1: int, _col2: int
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col2 ASC NULLS LAST
+ partition by: _col0
+ raw input shape:
+ window functions:
+ window function definition
+ alias: rank_window_0
+ arguments: _col2
+ name: rank
+ window function: GenericUDAFRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: dense_rank_window_1
+ arguments: _col2
+ name: dense_rank
+ window function: GenericUDAFDenseRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: cume_dist_window_2
+ arguments: _col2
+ name: cume_dist
+ window function: GenericUDAFCumeDistEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorRank,
VectorPTFEvaluatorDenseRank, VectorPTFEvaluatorCumeDist]
+ functionInputExpressions: [col 1:int, col 1:int, col
1:int]
+ functionNames: [rank, dense_rank, cume_dist]
+ keyInputColumns: [0, 1]
+ native: true
+ nonKeyInputColumns: [2]
+ orderExpressions: [col 1:int]
+ outputColumns: [3, 4, 5, 0, 2, 1]
+ outputTypes: [int, int, double, string, int, int]
+ partitionExpressions: [col 0:string]
+ streamingColumns: [3, 4, 5]
+ Statistics: Num rows: 24 Data size: 2227 Basic stats:
COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: int),
_col2 (type: int), rank_window_0 (type: int), dense_rank_window_1 (type: int),
cume_dist_window_2 (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 2, 1, 3, 4, 5]
+ Statistics: Num rows: 24 Data size: 2417 Basic stats:
COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 24 Data size: 2417 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select name, rowindex, mynumber,
+rank() over (partition by name order by mynumber) as r,
+dense_rank() over (partition by name order by mynumber) as dr,
+cume_dist() over (partition by name order by mynumber) as cud
+from vector_ptf_cume_dist_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_cume_dist_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+rank() over (partition by name order by mynumber) as r,
+dense_rank() over (partition by name order by mynumber) as dr,
+cume_dist() over (partition by name order by mynumber) as cud
+from vector_ptf_cume_dist_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_cume_dist_int
+#### A masked pattern was here ####
+NULL 44 7 1 1 1.0
+NULL 43 7 1 1 1.0
+first 1 1 1 1 0.25
+first 2 1 1 1 0.25
+first 3 1 1 1 0.25
+first 4 2 4 2 0.3333333333333333
+first 6 3 5 3 0.5
+first 5 3 5 3 0.5
+first 8 4 7 4 0.6666666666666666
+first 7 4 7 4 0.6666666666666666
+first 9 5 9 5 0.8333333333333334
+first 10 5 9 5 0.8333333333333334
+first 11 NULL 11 6 1.0
+first 12 NULL 11 6 1.0
+second 22 10 1 1 0.2
+second 23 10 1 1 0.2
+second 25 20 3 2 0.4
+second 24 20 3 2 0.4
+second 26 30 5 3 0.5
+second 27 40 6 4 0.7
+second 28 40 6 4 0.7
+second 29 50 8 5 0.8
+second 30 NULL 9 6 1.0
+second 31 NULL 9 6 1.0
+PREHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+rank() over (order by mynumber) as r,
+dense_rank() over (order by mynumber) as dr,
+cume_dist() over (order by mynumber) as cud
+from vector_ptf_cume_dist_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_cume_dist_int
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+rank() over (order by mynumber) as r,
+dense_rank() over (order by mynumber) as dr,
+cume_dist() over (order by mynumber) as cud
+from vector_ptf_cume_dist_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_cume_dist_int
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: vector_ptf_cume_dist_int
+ Statistics: Num rows: 24 Data size: 2227 Basic stats:
COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:name:string,
1:rowindex:int, 2:mynumber:int,
3:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>,
4:ROW__IS__DELETED:boolean]
+ Reduce Output Operator
+ key expressions: 0 (type: int), mynumber (type: int)
+ null sort order: az
+ sort order: ++
+ Map-reduce partition columns: 0 (type: int)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: 5:int, 2:int
+ keyExpressions: ConstantVectorExpression(val 0) ->
5:int
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ partitionColumns: 6:int
+ valueColumns: 0:string, 1:int
+ Statistics: Num rows: 24 Data size: 2227 Basic stats:
COMPLETE Column stats: COMPLETE
+ value expressions: name (type: string), rowindex (type:
int)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet:
hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats:
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ includeColumns: [0, 1, 2]
+ dataColumns: name:string, rowindex:int, mynumber:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, bigint]
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: az
+ reduceColumnSortOrder: ++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 4
+ dataColumns: KEY.reducesinkkey0:int,
KEY.reducesinkkey1:int, VALUE._col0:string, VALUE._col1:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, bigint, double, bigint]
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string), VALUE._col1 (type:
int), KEY.reducesinkkey1 (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [2, 3, 1]
+ Statistics: Num rows: 24 Data size: 2227 Basic stats: COMPLETE
Column stats: COMPLETE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string, _col1: int, _col2: int
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col2 ASC NULLS LAST
+ partition by: 0
+ raw input shape:
+ window functions:
+ window function definition
+ alias: rank_window_0
+ arguments: _col2
+ name: rank
+ window function: GenericUDAFRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: dense_rank_window_1
+ arguments: _col2
+ name: dense_rank
+ window function: GenericUDAFDenseRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: cume_dist_window_2
+ arguments: _col2
+ name: cume_dist
+ window function: GenericUDAFCumeDistEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorRank,
VectorPTFEvaluatorDenseRank, VectorPTFEvaluatorCumeDist]
+ functionInputExpressions: [col 1:int, col 1:int, col
1:int]
+ functionNames: [rank, dense_rank, cume_dist]
+ keyInputColumns: [1]
+ native: true
+ nonKeyInputColumns: [2, 3]
+ orderExpressions: [col 1:int]
+ outputColumns: [4, 5, 6, 2, 3, 1]
+ outputTypes: [int, int, double, string, int, int]
+ partitionExpressions: [ConstantVectorExpression(val 0)
-> 7:int]
+ streamingColumns: [4, 5, 6]
+ Statistics: Num rows: 24 Data size: 2227 Basic stats:
COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: int),
_col2 (type: int), rank_window_0 (type: int), dense_rank_window_1 (type: int),
cume_dist_window_2 (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [2, 3, 1, 4, 5, 6]
+ Statistics: Num rows: 24 Data size: 2417 Basic stats:
COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 24 Data size: 2417 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select name, rowindex, mynumber,
+rank() over (order by mynumber) as r,
+dense_rank() over (order by mynumber) as dr,
+cume_dist() over (order by mynumber) as cud
+from vector_ptf_cume_dist_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_cume_dist_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+rank() over (order by mynumber) as r,
+dense_rank() over (order by mynumber) as dr,
+cume_dist() over (order by mynumber) as cud
+from vector_ptf_cume_dist_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_cume_dist_int
+#### A masked pattern was here ####
+first 1 1 1 1 0.125
+first 2 1 1 1 0.125
+first 3 1 1 1 0.125
+first 4 2 4 2 0.16666666666666666
+first 5 3 5 3 0.25
+first 6 3 5 3 0.25
+first 7 4 7 4 0.3333333333333333
+first 8 4 7 4 0.3333333333333333
+first 9 5 9 5 0.4166666666666667
+first 10 5 9 5 0.4166666666666667
+NULL 44 7 11 6 0.5
+NULL 43 7 11 6 0.5
+second 23 10 13 7 0.5833333333333334
+second 22 10 13 7 0.5833333333333334
+second 24 20 15 8 0.6666666666666666
+second 25 20 15 8 0.6666666666666666
+second 26 30 17 9 0.7083333333333334
+second 27 40 18 10 0.7916666666666666
+second 28 40 18 10 0.7916666666666666
+second 29 50 20 11 0.8333333333333334
+first 11 NULL 21 12 1.0
+second 30 NULL 21 12 1.0
+second 31 NULL 21 12 1.0
+first 12 NULL 21 12 1.0
+PREHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+rank() over (partition by name) as r,
+dense_rank() over (partition by name) as dr,
+cume_dist() over (partition by name) as cud
+from vector_ptf_cume_dist_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_cume_dist_int
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+rank() over (partition by name) as r,
+dense_rank() over (partition by name) as dr,
+cume_dist() over (partition by name) as cud
+from vector_ptf_cume_dist_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_cume_dist_int
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: vector_ptf_cume_dist_int
+ Statistics: Num rows: 24 Data size: 2227 Basic stats:
COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:name:string,
1:rowindex:int, 2:mynumber:int,
3:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>,
4:ROW__IS__DELETED:boolean]
+ Reduce Output Operator
+ key expressions: name (type: string)
+ null sort order: a
+ sort order: +
+ Map-reduce partition columns: name (type: string)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkStringOperator
+ keyColumns: 0:string
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 1:int, 2:int
+ Statistics: Num rows: 24 Data size: 2227 Basic stats:
COMPLETE Column stats: COMPLETE
+ value expressions: rowindex (type: int), mynumber (type:
int)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet:
hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats:
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ includeColumns: [0, 1, 2]
+ dataColumns: name:string, rowindex:int, mynumber:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: a
+ reduceColumnSortOrder: +
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ dataColumns: KEY.reducesinkkey0:string, VALUE._col0:int,
VALUE._col1:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, bigint, double]
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string), VALUE._col0
(type: int), VALUE._col1 (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1, 2]
+ Statistics: Num rows: 24 Data size: 2227 Basic stats: COMPLETE
Column stats: COMPLETE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string, _col1: int, _col2: int
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col0 ASC NULLS FIRST
+ partition by: _col0
+ raw input shape:
+ window functions:
+ window function definition
+ alias: rank_window_0
+ arguments: _col0
+ name: rank
+ window function: GenericUDAFRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: dense_rank_window_1
+ arguments: _col0
+ name: dense_rank
+ window function: GenericUDAFDenseRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: cume_dist_window_2
+ arguments: _col0
+ name: cume_dist
+ window function: GenericUDAFCumeDistEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorRank,
VectorPTFEvaluatorDenseRank, VectorPTFEvaluatorCumeDist]
+ functionInputExpressions: [col 0:string, col 0:string,
col 0:string]
+ functionNames: [rank, dense_rank, cume_dist]
+ keyInputColumns: [0]
+ native: true
+ nonKeyInputColumns: [1, 2]
+ orderExpressions: [col 0:string]
+ outputColumns: [3, 4, 5, 0, 1, 2]
+ outputTypes: [int, int, double, string, int, int]
+ partitionExpressions: [col 0:string]
+ streamingColumns: [3, 4, 5]
+ Statistics: Num rows: 24 Data size: 2227 Basic stats:
COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: int),
_col2 (type: int), rank_window_0 (type: int), dense_rank_window_1 (type: int),
cume_dist_window_2 (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1, 2, 3, 4, 5]
+ Statistics: Num rows: 24 Data size: 2417 Basic stats:
COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 24 Data size: 2417 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select name, rowindex, mynumber,
+rank() over (partition by name) as r,
+dense_rank() over (partition by name) as dr,
+cume_dist() over (partition by name) as cud
+from vector_ptf_cume_dist_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_cume_dist_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+rank() over (partition by name) as r,
+dense_rank() over (partition by name) as dr,
+cume_dist() over (partition by name) as cud
+from vector_ptf_cume_dist_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_cume_dist_int
+#### A masked pattern was here ####
+NULL 44 7 1 1 1.0
+NULL 43 7 1 1 1.0
+first 1 1 1 1 1.0
+first 2 1 1 1 1.0
+first 3 1 1 1 1.0
+first 4 2 1 1 1.0
+first 5 3 1 1 1.0
+first 6 3 1 1 1.0
+first 7 4 1 1 1.0
+first 8 4 1 1 1.0
+first 9 5 1 1 1.0
+first 10 5 1 1 1.0
+first 11 NULL 1 1 1.0
+first 12 NULL 1 1 1.0
+second 24 20 1 1 1.0
+second 25 20 1 1 1.0
+second 26 30 1 1 1.0
+second 27 40 1 1 1.0
+second 28 40 1 1 1.0
+second 29 50 1 1 1.0
+second 30 NULL 1 1 1.0
+second 31 NULL 1 1 1.0
+second 23 10 1 1 1.0
+second 22 10 1 1 1.0
+PREHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+rank() over () as r,
+dense_rank() over () as dr,
+cume_dist() over () as cud
+from vector_ptf_cume_dist_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_cume_dist_int
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+rank() over () as r,
+dense_rank() over () as dr,
+cume_dist() over () as cud
+from vector_ptf_cume_dist_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_cume_dist_int
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: vector_ptf_cume_dist_int
+ Statistics: Num rows: 24 Data size: 2227 Basic stats:
COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:name:string,
1:rowindex:int, 2:mynumber:int,
3:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>,
4:ROW__IS__DELETED:boolean]
+ Reduce Output Operator
+ key expressions: 0 (type: int)
+ null sort order: a
+ sort order: +
+ Map-reduce partition columns: 0 (type: int)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkLongOperator
+ keyColumns: 5:int
+ keyExpressions: ConstantVectorExpression(val 0) ->
5:int
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 0:string, 1:int, 2:int
+ Statistics: Num rows: 24 Data size: 2227 Basic stats:
COMPLETE Column stats: COMPLETE
+ value expressions: name (type: string), rowindex (type:
int), mynumber (type: int)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet:
hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats:
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ includeColumns: [0, 1, 2]
+ dataColumns: name:string, rowindex:int, mynumber:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint]
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: a
+ reduceColumnSortOrder: +
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 4
+ dataColumns: KEY.reducesinkkey0:int, VALUE._col0:string,
VALUE._col1:int, VALUE._col2:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, bigint, double, bigint,
bigint, bigint, bigint, bigint]
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string), VALUE._col1 (type:
int), VALUE._col2 (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [1, 2, 3]
+ Statistics: Num rows: 24 Data size: 2227 Basic stats: COMPLETE
Column stats: COMPLETE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string, _col1: int, _col2: int
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: 0 ASC NULLS FIRST
+ partition by: 0
+ raw input shape:
+ window functions:
+ window function definition
+ alias: rank_window_0
+ arguments: 0
+ name: rank
+ window function: GenericUDAFRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: dense_rank_window_1
+ arguments: 0
+ name: dense_rank
+ window function: GenericUDAFDenseRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: cume_dist_window_2
+ arguments: 0
+ name: cume_dist
+ window function: GenericUDAFCumeDistEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorRank,
VectorPTFEvaluatorDenseRank, VectorPTFEvaluatorCumeDist]
+ functionInputExpressions: [ConstantVectorExpression(val
0) -> 9:int, ConstantVectorExpression(val 0) -> 10:int,
ConstantVectorExpression(val 0) -> 11:int]
+ functionNames: [rank, dense_rank, cume_dist]
+ keyInputColumns: []
+ native: true
+ nonKeyInputColumns: [1, 2, 3]
+ orderExpressions: [ConstantVectorExpression(val 0) ->
8:int]
+ outputColumns: [4, 5, 6, 1, 2, 3]
+ outputTypes: [int, int, double, string, int, int]
+ partitionExpressions: [ConstantVectorExpression(val 0)
-> 7:int]
+ streamingColumns: [4, 5, 6]
+ Statistics: Num rows: 24 Data size: 2227 Basic stats:
COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: int),
_col2 (type: int), rank_window_0 (type: int), dense_rank_window_1 (type: int),
cume_dist_window_2 (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [1, 2, 3, 4, 5, 6]
+ Statistics: Num rows: 24 Data size: 2417 Basic stats:
COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 24 Data size: 2417 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select name, rowindex, mynumber,
+rank() over () as r,
+dense_rank() over () as dr,
+cume_dist() over () as cud
+from vector_ptf_cume_dist_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_cume_dist_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+rank() over () as r,
+dense_rank() over () as dr,
+cume_dist() over () as cud
+from vector_ptf_cume_dist_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_cume_dist_int
+#### A masked pattern was here ####
+first 1 1 1 1 1.0
+first 2 1 1 1 1.0
+first 3 1 1 1 1.0
+first 4 2 1 1 1.0
+first 5 3 1 1 1.0
+first 6 3 1 1 1.0
+first 7 4 1 1 1.0
+first 8 4 1 1 1.0
+first 9 5 1 1 1.0
+first 10 5 1 1 1.0
+first 11 NULL 1 1 1.0
+first 12 NULL 1 1 1.0
+second 22 10 1 1 1.0
+second 23 10 1 1 1.0
+second 24 20 1 1 1.0
+second 25 20 1 1 1.0
+second 26 30 1 1 1.0
+second 27 40 1 1 1.0
+second 28 40 1 1 1.0
+second 29 50 1 1 1.0
+second 30 NULL 1 1 1.0
+second 31 NULL 1 1 1.0
+NULL 43 7 1 1 1.0
+NULL 44 7 1 1 1.0
diff --git a/ql/src/test/results/clientpositive/llap/vector_windowing.q.out
b/ql/src/test/results/clientpositive/llap/vector_windowing.q.out
index 2b3c63670fb..5ba2c7af513 100644
--- a/ql/src/test/results/clientpositive/llap/vector_windowing.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_windowing.q.out
@@ -3250,7 +3250,7 @@ STAGE PLANS:
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
- notVectorizedReason: PTF operator: cume_dist not in supported
functions [avg, count, dense_rank, first_value, lag, last_value, lead, max,
min, rank, row_number, sum]
+ notVectorizedReason: PTF operator: percent_rank not in
supported functions [avg, count, cume_dist, dense_rank, first_value, lag,
last_value, lead, max, min, rank, row_number, sum]
vectorized: false
Reduce Operator Tree:
Select Operator
@@ -3543,16 +3543,28 @@ STAGE PLANS:
partitionColumnCount: 0
scratchColumnTypeNames: []
Reducer 2
- Execution mode: llap
+ Execution mode: vectorized, llap
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
- notVectorizedReason: PTF operator: cume_dist not in supported
functions [avg, count, dense_rank, first_value, lag, last_value, lead, max,
min, rank, row_number, sum]
- vectorized: false
+ reduceColumnNullOrder: az
+ reduceColumnSortOrder: ++
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ dataColumns: KEY.reducesinkkey0:string,
KEY.reducesinkkey1:string, VALUE._col3:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, bigint, double, bigint]
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey1 (type: string),
KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int)
outputColumnNames: _col1, _col2, _col5
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [1, 0, 2]
Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE
Column stats: COMPLETE
PTF Operator
Function definitions:
@@ -3594,16 +3606,41 @@ STAGE PLANS:
name: sum
window function: GenericUDAFSumLong
window frame: RANGE PRECEDING(MAX)~CURRENT
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorRank,
VectorPTFEvaluatorDenseRank, VectorPTFEvaluatorCumeDist,
VectorPTFEvaluatorLongSum]
+ functionInputExpressions: [col 1:string, col 1:string,
col 1:string, col 2:int]
+ functionNames: [rank, dense_rank, cume_dist, sum]
+ keyInputColumns: [1, 0]
+ native: true
+ nonKeyInputColumns: [2]
+ orderExpressions: [col 1:string]
+ outputColumns: [3, 4, 5, 6, 1, 0, 2]
+ outputTypes: [int, int, double, bigint, string, string,
int]
+ partitionExpressions: [col 0:string]
+ streamingColumns: [3, 4, 5]
Statistics: Num rows: 26 Data size: 5798 Basic stats:
COMPLETE Column stats: COMPLETE
Select Operator
expressions: rank_window_0 (type: int),
dense_rank_window_1 (type: int), cume_dist_window_2 (type: double),
sum_window_3 (type: bigint), _col1 (type: string), _col2 (type: string), _col5
(type: int)
outputColumnNames: rank_window_0, dense_rank_window_1,
cume_dist_window_2, sum_window_3, _col1, _col2, _col5
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [3, 4, 5, 6, 1, 0, 2]
Statistics: Num rows: 26 Data size: 5798 Basic stats:
COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col2 (type: string), _col5 (type: int)
null sort order: az
sort order: ++
Map-reduce partition columns: _col2 (type: string)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: 0:string, 2:int
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ partitionColumns: 0:string
+ valueColumns: 3:int, 4:int, 5:double, 6:bigint,
1:string
Statistics: Num rows: 26 Data size: 5798 Basic stats:
COMPLETE Column stats: COMPLETE
value expressions: rank_window_0 (type: int),
dense_rank_window_1 (type: int), cume_dist_window_2 (type: double),
sum_window_3 (type: bigint), _col1 (type: string)
Reducer 3
@@ -6166,16 +6203,28 @@ STAGE PLANS:
output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Reducer 4
- Execution mode: llap
+ Execution mode: vectorized, llap
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
- notVectorizedReason: PTF operator: cume_dist not in supported
functions [avg, count, dense_rank, first_value, lag, last_value, lead, max,
min, rank, row_number, sum]
- vectorized: false
+ reduceColumnNullOrder: az
+ reduceColumnSortOrder: ++
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ dataColumns: KEY.reducesinkkey0:string,
KEY.reducesinkkey1:string, VALUE._col3:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, bigint, double]
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey1 (type: string),
KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int)
outputColumnNames: _col1, _col2, _col5
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [1, 0, 2]
Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE
Column stats: COMPLETE
PTF Operator
Function definitions:
@@ -6211,16 +6260,41 @@ STAGE PLANS:
window function: GenericUDAFCumeDistEvaluator
window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
isPivotResult: true
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorRank,
VectorPTFEvaluatorDenseRank, VectorPTFEvaluatorCumeDist]
+ functionInputExpressions: [col 1:string, col 1:string,
col 1:string]
+ functionNames: [rank, dense_rank, cume_dist]
+ keyInputColumns: [1, 0]
+ native: true
+ nonKeyInputColumns: [2]
+ orderExpressions: [col 1:string]
+ outputColumns: [3, 4, 5, 1, 0, 2]
+ outputTypes: [int, int, double, string, string, int]
+ partitionExpressions: [col 0:string]
+ streamingColumns: [3, 4, 5]
Statistics: Num rows: 26 Data size: 5798 Basic stats:
COMPLETE Column stats: COMPLETE
Select Operator
expressions: rank_window_0 (type: int),
dense_rank_window_1 (type: int), cume_dist_window_2 (type: double), _col1
(type: string), _col2 (type: string), _col5 (type: int)
outputColumnNames: rank_window_0, dense_rank_window_1,
cume_dist_window_2, _col1, _col2, _col5
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [3, 4, 5, 1, 0, 2]
Statistics: Num rows: 26 Data size: 5798 Basic stats:
COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col2 (type: string), _col5 (type: int)
null sort order: az
sort order: ++
Map-reduce partition columns: _col2 (type: string)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: 0:string, 2:int
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ partitionColumns: 0:string
+ valueColumns: 3:int, 4:int, 5:double, 1:string
Statistics: Num rows: 26 Data size: 5798 Basic stats:
COMPLETE Column stats: COMPLETE
value expressions: rank_window_0 (type: int),
dense_rank_window_1 (type: int), cume_dist_window_2 (type: double), _col1
(type: string)
Reducer 5
diff --git
a/ql/src/test/results/clientpositive/llap/vector_windowing_gby2.q.out
b/ql/src/test/results/clientpositive/llap/vector_windowing_gby2.q.out
index 239b6969d23..c2faf5d03a6 100644
--- a/ql/src/test/results/clientpositive/llap/vector_windowing_gby2.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_windowing_gby2.q.out
@@ -796,7 +796,7 @@ STAGE PLANS:
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
- notVectorizedReason: PTF operator: percent_rank not in
supported functions [avg, count, dense_rank, first_value, lag, last_value,
lead, max, min, rank, row_number, sum]
+ notVectorizedReason: PTF operator: percent_rank not in
supported functions [avg, count, cume_dist, dense_rank, first_value, lag,
last_value, lead, max, min, rank, row_number, sum]
vectorized: false
Reduce Operator Tree:
Select Operator
diff --git
a/ql/src/test/results/clientpositive/llap/vector_windowing_rank.q.out
b/ql/src/test/results/clientpositive/llap/vector_windowing_rank.q.out
index 8abc2947cf6..45ab7990da0 100644
--- a/ql/src/test/results/clientpositive/llap/vector_windowing_rank.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_windowing_rank.q.out
@@ -639,16 +639,28 @@ STAGE PLANS:
partitionColumnCount: 0
scratchColumnTypeNames: []
Reducer 2
- Execution mode: llap
+ Execution mode: vectorized, llap
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
- notVectorizedReason: PTF operator: cume_dist not in supported
functions [avg, count, dense_rank, first_value, lag, last_value, lead, max,
min, rank, row_number, sum]
- vectorized: false
+ reduceColumnNullOrder: azz
+ reduceColumnSortOrder: +++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ dataColumns: KEY.reducesinkkey0:boolean,
KEY.reducesinkkey1:bigint, KEY.reducesinkkey2:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [double]
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey1 (type: bigint),
KEY.reducesinkkey0 (type: boolean), KEY.reducesinkkey2 (type: string)
outputColumnNames: _col3, _col6, _col7
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [1, 0, 2]
Statistics: Num rows: 1 Data size: 196 Basic stats: COMPLETE
Column stats: NONE
PTF Operator
Function definitions:
@@ -670,16 +682,40 @@ STAGE PLANS:
window function: GenericUDAFCumeDistEvaluator
window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
isPivotResult: true
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorCumeDist]
+ functionInputExpressions: [col 1:bigint]
+ functionNames: [cume_dist]
+ keyInputColumns: [1, 0, 2]
+ native: true
+ nonKeyInputColumns: []
+ orderExpressions: [col 1:bigint, col 2:string]
+ outputColumns: [3, 1, 0, 2]
+ outputTypes: [double, bigint, boolean, string]
+ partitionExpressions: [col 0:boolean]
+ streamingColumns: [3]
Statistics: Num rows: 1 Data size: 196 Basic stats: COMPLETE
Column stats: NONE
Limit
Number of rows: 100
+ Limit Vectorization:
+ className: VectorLimitOperator
+ native: true
Statistics: Num rows: 1 Data size: 196 Basic stats:
COMPLETE Column stats: NONE
Select Operator
expressions: _col7 (type: string), cume_dist_window_0
(type: double)
outputColumnNames: _col0, _col1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [2, 3]
Statistics: Num rows: 1 Data size: 196 Basic stats:
COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
Statistics: Num rows: 1 Data size: 196 Basic stats:
COMPLETE Column stats: NONE
table:
input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
@@ -872,7 +908,7 @@ STAGE PLANS:
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
- notVectorizedReason: PTF operator: percent_rank not in
supported functions [avg, count, dense_rank, first_value, lag, last_value,
lead, max, min, rank, row_number, sum]
+ notVectorizedReason: PTF operator: percent_rank not in
supported functions [avg, count, cume_dist, dense_rank, first_value, lag,
last_value, lead, max, min, rank, row_number, sum]
vectorized: false
Reduce Operator Tree:
Select Operator
diff --git a/ql/src/test/results/clientpositive/llap/vectorized_ptf.q.out
b/ql/src/test/results/clientpositive/llap/vectorized_ptf.q.out
index 1590dd76e4c..27bdbef3686 100644
--- a/ql/src/test/results/clientpositive/llap/vectorized_ptf.q.out
+++ b/ql/src/test/results/clientpositive/llap/vectorized_ptf.q.out
@@ -4531,7 +4531,7 @@ STAGE PLANS:
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
- notVectorizedReason: PTF operator: cume_dist not in supported
functions [avg, count, dense_rank, first_value, lag, last_value, lead, max,
min, rank, row_number, sum]
+ notVectorizedReason: PTF operator: first_value UNBOUNDED end
frame is required for ROWS window type
vectorized: false
Reduce Operator Tree:
Select Operator
