This is an automated email from the ASF dual-hosted git repository.
abstractdog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new ce171751a6a HIVE-30018: PTF: Vectorize ntile function (#6861)
ce171751a6a is described below
commit ce171751a6a99c3f54a32a1963d708b18d5edcfb
Author: gopigowtham <[email protected]>
AuthorDate: Thu Oct 15 11:48:12 2026 +0530
HIVE-30018: PTF: Vectorize ntile function (#6861)
---
.../exec/vector/ptf/VectorPTFEvaluatorNTile.java | 152 ++
.../hive/ql/exec/vector/ptf/VectorPTFOperator.java | 30 +-
.../hive/ql/optimizer/physical/Vectorizer.java | 13 +
.../apache/hadoop/hive/ql/plan/VectorPTFDesc.java | 9 +-
.../clientpositive/sketches_rewrite_cume_dist.q | 4 -
.../sketches_rewrite_cume_dist_partition_by.q | 4 -
.../vector_ptf_buffered_complex_scratch_col.q | 50 +
.../test/queries/clientpositive/vector_ptf_ntile.q | 151 ++
.../llap/sketches_materialized_view_ntile.q.out | 2 +-
.../llap/sketches_rewrite_ntile.q.out | 2 +-
.../vector_ptf_buffered_complex_scratch_col.q.out | 291 ++++
.../clientpositive/llap/vector_ptf_ntile.q.out | 1692 ++++++++++++++++++++
.../clientpositive/llap/vector_windowing.q.out | 2 +-
13 files changed, 2385 insertions(+), 17 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorNTile.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorNTile.java
new file mode 100644
index 00000000000..f99fa41af64
--- /dev/null
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorNTile.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.ptf;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hadoop.hive.ql.exec.vector.ColumnVector.Type;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import
org.apache.hadoop.hive.ql.exec.vector.expressions.ConstantVectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.ptf.WindowFrameDef;
+
+/**
+ * Evaluates {@code ntile(n)} as a <b>group-aggregated streaming</b> evaluator
(see
+ * {@link VectorPTFEvaluatorBase}).
+ *
+ * <p>NTILE divides the rows of a partition into {@code n} buckets and returns
the 1-based bucket
+ * number of the current row. When the partition size is not divisible by
{@code n}, the first
+ * {@code partitionSize % n} buckets each get one extra row, matching {@code
GenericUDAFNTile}.
+ * The result depends only on row position and partition size, never on peer
groups.
+ *
+ * <p>The partition is buffered so {@link #setPartitionSize(int)} is known
before output is
+ * written. Bucket boundaries are then derived on the fly while the batches
are replayed, so the
+ * evaluator keeps O(1) state regardless of partition size, and a batch that
falls entirely within
+ * one bucket is emitted as a repeating vector.
+ *
+ * <p>The number of buckets must be a positive integer constant (e.g. {@code
ntile(4)}); the
+ * Vectorizer leaves any other form to row mode.
+ */
+public class VectorPTFEvaluatorNTile extends VectorPTFEvaluatorBase {
+
+ private final long numBuckets;
+
+ // Per-partition bucket layout, set up by addStreamingGroupResults.
+ private int bucketSize;
+ private int largerBucketCount;
+
+ // Replay position within the partition.
+ private int currentBucket;
+ private int rowsLeftInBucket;
+
+ public VectorPTFEvaluatorNTile(WindowFrameDef windowFrameDef,
+ VectorExpression numBucketsExpression, int outputColumnNum) {
+ super(windowFrameDef, outputColumnNum);
+ if (!(numBucketsExpression instanceof ConstantVectorExpression
constantExpr)) {
+ throw new RuntimeException(
+ "NTILE expects a constant integer expression for the number of
buckets, got: "
+ + (numBucketsExpression == null ? "null"
+ : numBucketsExpression.getClass().getName()));
+ }
+ numBuckets = constantExpr.getLongValue();
+ if (numBuckets <= 0) {
+ throw new RuntimeException(
+ "NTILE requires a positive number of buckets, got: " + numBuckets);
+ }
+ resetEvaluator();
+ }
+
+ @Override
+ public boolean isGroupAggregatedStreamingEvaluator() {
+ return true;
+ }
+
+ @Override
+ public void addStreamingGroupResults(List<Integer> groupRowCounts) throws
HiveException {
+ if (partitionSize <= 0) {
+ throw new HiveException("Partition size must be set before computing
ntile");
+ }
+ // Peer groups are irrelevant: buckets depend only on row position and
partition size.
+ // With more buckets than rows every row gets its own bucket, which is
ntile(partitionSize);
+ // capping the bucket count keeps every bucket non-empty (and fits it in
an int).
+ final int buckets = (int) Math.min(numBuckets, partitionSize);
+ bucketSize = partitionSize / buckets;
+ largerBucketCount = partitionSize % buckets;
+ currentBucket = 1;
+ rowsLeftInBucket = rowsInBucket(currentBucket);
+ }
+
+ @Override
+ public void evaluateGroupBatch(VectorizedRowBatch batch) throws
HiveException {
+ final int size = batch.size;
+ LongColumnVector outputColVector = (LongColumnVector)
batch.cols[outputColumnNum];
+ outputColVector.noNulls = true;
+
+ if (size <= rowsLeftInBucket) {
+ // The whole batch falls inside the current bucket.
+ outputColVector.isRepeating = true;
+ outputColVector.isNull[0] = false;
+ outputColVector.vector[0] = currentBucket;
+ consumeRows(size);
+ return;
+ }
+
+ outputColVector.isRepeating = false;
+ long[] vector = outputColVector.vector;
+ int i = 0;
+ while (i < size) {
+ // Buckets are never empty, so every run covers at least one row.
+ int run = Math.min(rowsLeftInBucket, size - i);
+ Arrays.fill(vector, i, i + run, currentBucket);
+ i += run;
+ consumeRows(run);
+ }
+ }
+
+ private void consumeRows(int rows) {
+ rowsLeftInBucket -= rows;
+ if (rowsLeftInBucket == 0) {
+ currentBucket++;
+ rowsLeftInBucket = rowsInBucket(currentBucket);
+ }
+ }
+
+ private int rowsInBucket(int bucket) {
+ return bucket <= largerBucketCount ? bucketSize + 1 : bucketSize;
+ }
+
+ @Override
+ public boolean streamsResult() {
+ return true;
+ }
+
+ @Override
+ public Type getResultColumnVectorType() {
+ return Type.LONG;
+ }
+
+ @Override
+ public void resetEvaluator() {
+ partitionSize = -1;
+ }
+}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFOperator.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFOperator.java
index 38e9a6d911f..76a5affbe31 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFOperator.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFOperator.java
@@ -22,6 +22,7 @@
import java.sql.Timestamp;
import java.util.Arrays;
import java.util.List;
+import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
@@ -435,10 +436,11 @@ private void initBufferedColumns() {
* 1. order columns
* 2. the non-key input columns
* 3. any streaming columns that will already have their output values
- * 4. scratch columns (as they're used by input expressions of evaluators)
+ * 4. scratch columns holding the results of evaluator input expressions
*/
+ final int[] bufferedScratchIndices = getEvaluatorInputScratchIndices();
int bufferedColumnCount = orderColumnMap.length +
nonKeyInputColumnMap.length
- + streamingEvaluatorNums.length + scratchColumnPositions.length;
+ + streamingEvaluatorNums.length + bufferedScratchIndices.length;
this.bufferedColumnMap = new int[bufferedColumnCount];
this.bufferedTypeInfos = new TypeInfo[bufferedColumnCount];
@@ -474,13 +476,31 @@ private void initBufferedColumns() {
bufferedTypeInfos[bufferedMapIndex] =
outputTypeInfos[streamingEvaluatorNum];
}
- for (int i = 0; i < scratchColumnPositions.length; i++) {
+ for (int i = 0; i < bufferedScratchIndices.length; i++) {
+ final int scratchIndex = bufferedScratchIndices[i];
final int bufferedMapIndex = orderColumnCount + nonKeyInputColumnCount +
streamingEvaluatorCount + i;
- bufferedColumnMap[bufferedMapIndex] = scratchColumnPositions[i];
- bufferedTypeInfos[bufferedMapIndex] =
TypeInfoUtils.getTypeInfoFromTypeString(vOutContext.getScratchColumnTypeNames()[i]);
+ bufferedColumnMap[bufferedMapIndex] =
scratchColumnPositions[scratchIndex];
+ bufferedTypeInfos[bufferedMapIndex] =
+
TypeInfoUtils.getTypeInfoFromTypeString(vOutContext.getScratchColumnTypeNames()[scratchIndex]);
}
}
+ /**
+ * Returns the indices (into scratchColumnPositions) of the scratch columns
written by evaluator
+ * input expressions. Those are evaluated before a batch is buffered, so
their results must be
+ * buffered too. The other scratch columns belong to child operators, which
recompute them after
+ * the batch is forwarded; buffering them is wasted work and fails for
complex types.
+ */
+ private int[] getEvaluatorInputScratchIndices() {
+ final Set<Integer> inputExprColumns = Arrays.stream(evaluators)
+ .filter(evaluator -> evaluator.inputVecExpr != null)
+ .map(evaluator -> evaluator.inputVecExpr.getOutputColumnNum())
+ .collect(Collectors.toSet());
+ return IntStream.range(0, scratchColumnPositions.length)
+ .filter(i -> inputExprColumns.contains(scratchColumnPositions[i]))
+ .toArray();
+ }
+
private void initExpressionColumns() {
for (VectorPTFEvaluatorBase evaluator : evaluators) {
/*
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
index 36cc904c261..3be1c060a94 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
@@ -125,6 +125,7 @@
import org.apache.hadoop.hive.ql.plan.BaseWork;
import org.apache.hadoop.hive.ql.plan.Explain;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc.ExprNodeDescEqualityWrapper;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
@@ -3011,6 +3012,13 @@ private boolean validatePTFOperator(PTFOperator op,
VectorizationContext vContex
throw new RuntimeException("Unexpected window type " +
windowFrameDef.getWindowType());
}
+ // Row mode also accepts a non-constant ntile argument (read from the
first row of the partition).
+ if (supportedFunctionType == SupportedFunctionType.NTILE &&
+ !isPositiveIntConstant(singleExprNodeDesc)) {
+ setOperatorIssue(functionName + " only supports a positive integer
constant number of buckets");
+ return false;
+ }
+
if
(!VectorPTFDesc.COLUMN_AGNOSTIC_FUNCTIONS.contains(supportedFunctionType)) {
if (exprNodeDescList != null) {
@@ -5042,6 +5050,11 @@ private static ExprNodeDesc[]
getOrderExprNodeDescs(List<OrderExpressionDef> ord
return exprNodeDescs;
}
+ private static boolean isPositiveIntConstant(ExprNodeDesc exprNodeDesc) {
+ return exprNodeDesc instanceof ExprNodeConstantDesc constantDesc
+ && constantDesc.getValue() instanceof Integer value && value > 0;
+ }
+
// TODO: An evaluator that wants to handle an unbuffered partition-only
column in its calculation could
// opt in to vectorization here.
private static boolean hasUnbufferedPartitionColumnInEvaluatorArgs(
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorPTFDesc.java
b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorPTFDesc.java
index 38f730049dd..8a32328147c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorPTFDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorPTFDesc.java
@@ -60,6 +60,7 @@
import org.apache.hadoop.hive.ql.exec.vector.ptf.VectorPTFEvaluatorLongMax;
import org.apache.hadoop.hive.ql.exec.vector.ptf.VectorPTFEvaluatorLongMin;
import org.apache.hadoop.hive.ql.exec.vector.ptf.VectorPTFEvaluatorLongSum;
+import org.apache.hadoop.hive.ql.exec.vector.ptf.VectorPTFEvaluatorNTile;
import org.apache.hadoop.hive.ql.exec.vector.ptf.VectorPTFEvaluatorPercentRank;
import org.apache.hadoop.hive.ql.exec.vector.ptf.VectorPTFEvaluatorRank;
import org.apache.hadoop.hive.ql.exec.vector.ptf.VectorPTFEvaluatorRowNumber;
@@ -98,6 +99,7 @@ public enum SupportedFunctionType {
DENSE_RANK,
PERCENT_RANK,
CUME_DIST,
+ NTILE,
MIN,
MAX,
SUM,
@@ -143,7 +145,8 @@ public boolean isSupportDistinct() {
SupportedFunctionType.RANK,
SupportedFunctionType.DENSE_RANK,
SupportedFunctionType.PERCENT_RANK,
- SupportedFunctionType.CUME_DIST);
+ SupportedFunctionType.CUME_DIST,
+ SupportedFunctionType.NTILE);
private TypeInfo[] reducerBatchTypeInfos;
private DataTypePhysicalVariation[] reducerBatchDataTypePhysicalVariations;
@@ -222,6 +225,10 @@ public static VectorPTFEvaluatorBase
getEvaluator(SupportedFunctionType function
case CUME_DIST:
evaluator = new VectorPTFEvaluatorCumeDist(windowFrameDef,
outputColumnNum);
break;
+ case NTILE:
+ evaluator = new VectorPTFEvaluatorNTile(windowFrameDef,
inputVectorExpression,
+ outputColumnNum);
+ break;
case MIN:
switch (columnVectorType) {
case LONG:
diff --git a/ql/src/test/queries/clientpositive/sketches_rewrite_cume_dist.q
b/ql/src/test/queries/clientpositive/sketches_rewrite_cume_dist.q
index a7ef8cb2285..33e0f6528a2 100644
--- a/ql/src/test/queries/clientpositive/sketches_rewrite_cume_dist.q
+++ b/ql/src/test/queries/clientpositive/sketches_rewrite_cume_dist.q
@@ -12,15 +12,11 @@ insert into table sketch_input values
select id,cume_dist() over (order by id) from sketch_input;
--- Vectorized PTF buffered path does not support complex-type passthrough
columns.
-set hive.vectorized.execution.ptf.enabled=false;
-
select id,cume_dist() over (order by id),ds_kll_cdf(ds, CAST(id AS FLOAT) )[0]
from sketch_input
join ( select ds_kll_sketch(cast(id as float)) as ds from sketch_input ) q
order by id;
-set hive.vectorized.execution.ptf.enabled=true;
set hive.optimize.bi.enabled=true;
-- see if rewrite happens
diff --git
a/ql/src/test/queries/clientpositive/sketches_rewrite_cume_dist_partition_by.q
b/ql/src/test/queries/clientpositive/sketches_rewrite_cume_dist_partition_by.q
index 2449c679ea2..6622bb3b8b8 100644
---
a/ql/src/test/queries/clientpositive/sketches_rewrite_cume_dist_partition_by.q
+++
b/ql/src/test/queries/clientpositive/sketches_rewrite_cume_dist_partition_by.q
@@ -10,15 +10,11 @@ insert into table sketch_input values
(1,null),(2,null),(10,null),(13,null);
;
--- Vectorized PTF buffered path does not support complex-type passthrough
columns.
-set hive.vectorized.execution.ptf.enabled=false;
-
select id,category,cume_dist() over (partition by category order by
id),ds_kll_cdf(ds, CAST(id AS FLOAT))[0]
from sketch_input
join ( select category as c,ds_kll_sketch(cast(id as float)) as ds from
sketch_input group by category) q on (q.c=category)
order by category,id;
-set hive.vectorized.execution.ptf.enabled=true;
set hive.optimize.bi.enabled=true;
-- see if rewrite happens
diff --git
a/ql/src/test/queries/clientpositive/vector_ptf_buffered_complex_scratch_col.q
b/ql/src/test/queries/clientpositive/vector_ptf_buffered_complex_scratch_col.q
new file mode 100644
index 00000000000..b91d3ef165c
--- /dev/null
+++
b/ql/src/test/queries/clientpositive/vector_ptf_buffered_complex_scratch_col.q
@@ -0,0 +1,50 @@
+set hive.explain.user=false;
+set hive.vectorized.testing.reducer.batch.size=2;
+
+DROP TABLE IF EXISTS vector_ptf_complex_scratch;
+
+CREATE TABLE vector_ptf_complex_scratch(name string, rowindex int, mynumber
int) stored as orc;
+
+INSERT INTO vector_ptf_complex_scratch values
+('first', 1, 1),
+('first', 2, 1),
+('first', 3, 2),
+('first', 4, 3),
+('first', 5, NULL),
+('second', 6, 10),
+('second', 7, 20),
+('second', 8, 20),
+(NULL, 9, 7);
+
+-- The PTF buffers each partition, while array(...) is computed by the Select
after the PTF.
+-- Its array<int> scratch column must not be buffered with the partition.
+set hive.vectorized.execution.ptf.enabled=false;
+
+select name, rowindex,
+ntile(2) over (partition by name order by rowindex) as nt,
+cume_dist() over (partition by name order by rowindex) as cd,
+sum(mynumber) over (partition by name order by rowindex) as s,
+array(rowindex, mynumber)[1] as arr
+from vector_ptf_complex_scratch;
+
+set hive.vectorized.execution.ptf.enabled=true;
+
+explain vectorization detail select name, rowindex,
+ntile(2) over (partition by name order by rowindex) as nt,
+cume_dist() over (partition by name order by rowindex) as cd,
+sum(mynumber) over (partition by name order by rowindex) as s,
+array(rowindex, mynumber)[1] as arr
+from vector_ptf_complex_scratch;
+
+select name, rowindex,
+ntile(2) over (partition by name order by rowindex) as nt,
+cume_dist() over (partition by name order by rowindex) as cd,
+sum(mynumber) over (partition by name order by rowindex) as s,
+array(rowindex, mynumber)[1] as arr
+from vector_ptf_complex_scratch;
+
+-- same shape without ntile: cume_dist alone also buffers the partition
+select name, rowindex,
+cume_dist() over (partition by name order by rowindex) as cd,
+array(rowindex, mynumber)[1] as arr
+from vector_ptf_complex_scratch;
diff --git a/ql/src/test/queries/clientpositive/vector_ptf_ntile.q
b/ql/src/test/queries/clientpositive/vector_ptf_ntile.q
new file mode 100644
index 00000000000..be2ef95d226
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/vector_ptf_ntile.q
@@ -0,0 +1,151 @@
+set hive.explain.user=false;
+set hive.vectorized.testing.reducer.batch.size=2;
+
+DROP TABLE IF EXISTS vector_ptf_ntile_int;
+
+CREATE TABLE vector_ptf_ntile_int(name string, rowindex int, mynumber int)
stored as orc;
+
+INSERT INTO vector_ptf_ntile_int values
+-- first partition (12 rows)
+('first', 1, 1),
+('first', 2, 1),
+('first', 3, 1),
+('first', 4, 2),
+('first', 5, 3),
+('first', 6, 3),
+('first', 7, 4),
+('first', 8, 4),
+('first', 9, 5),
+('first', 10, 5),
+('first', 11, NULL),
+('first', 12, NULL),
+-- second partition (10 rows)
+('second', 22, 10),
+('second', 23, 10),
+('second', 24, 20),
+('second', 25, 20),
+('second', 26, 30),
+('second', 27, 40),
+('second', 28, 40),
+('second', 29, 50),
+('second', 30, NULL),
+('second', 31, NULL),
+-- null partition (2 rows)
+(NULL, 43, 7),
+(NULL, 44, 7);
+
+-- NON-VECTORIZED: baseline output the vectorized run must match
+set hive.vectorized.execution.ptf.enabled=false;
+
+select name, rowindex, mynumber,
+ntile(4) over (partition by name order by mynumber) as nt
+from vector_ptf_ntile_int;
+
+select name, rowindex, mynumber,
+ntile(3) over (partition by name order by mynumber) as nt3,
+ntile(10) over (partition by name order by mynumber) as nt10
+from vector_ptf_ntile_int;
+
+select name, rowindex, mynumber,
+ntile(5) over (order by mynumber) as nt
+from vector_ptf_ntile_int;
+
+select name, rowindex, mynumber,
+ntile(4) over (partition by name) as nt
+from vector_ptf_ntile_int;
+
+select name, rowindex, mynumber,
+ntile(4) over () as nt
+from vector_ptf_ntile_int;
+
+select name, rowindex, mynumber,
+rank() over (partition by name order by mynumber) as r,
+cume_dist() over (partition by name order by mynumber) as cd,
+ntile(3) over (partition by name order by mynumber) as nt,
+sum(mynumber) over (partition by name order by mynumber) as s
+from vector_ptf_ntile_int;
+
+-- VECTORIZED: same queries with PTF vectorization on — results must be
identical
+set hive.vectorized.execution.ptf.enabled=true;
+
+explain vectorization detail select name, rowindex, mynumber,
+ntile(4) over (partition by name order by mynumber) as nt
+from vector_ptf_ntile_int;
+
+select name, rowindex, mynumber,
+ntile(4) over (partition by name order by mynumber) as nt
+from vector_ptf_ntile_int;
+
+explain vectorization detail select name, rowindex, mynumber,
+ntile(3) over (partition by name order by mynumber) as nt3,
+ntile(10) over (partition by name order by mynumber) as nt10
+from vector_ptf_ntile_int;
+
+select name, rowindex, mynumber,
+ntile(3) over (partition by name order by mynumber) as nt3,
+ntile(10) over (partition by name order by mynumber) as nt10
+from vector_ptf_ntile_int;
+
+explain vectorization detail select name, rowindex, mynumber,
+ntile(5) over (order by mynumber) as nt
+from vector_ptf_ntile_int;
+
+select name, rowindex, mynumber,
+ntile(5) over (order by mynumber) as nt
+from vector_ptf_ntile_int;
+
+explain vectorization detail select name, rowindex, mynumber,
+ntile(4) over (partition by name) as nt
+from vector_ptf_ntile_int;
+
+select name, rowindex, mynumber,
+ntile(4) over (partition by name) as nt
+from vector_ptf_ntile_int;
+
+explain vectorization detail select name, rowindex, mynumber,
+ntile(4) over () as nt
+from vector_ptf_ntile_int;
+
+select name, rowindex, mynumber,
+ntile(4) over () as nt
+from vector_ptf_ntile_int;
+
+-- ntile mixed with streaming (rank), group-aggregated streaming (cume_dist)
and buffered (sum) evaluators
+explain vectorization detail select name, rowindex, mynumber,
+rank() over (partition by name order by mynumber) as r,
+cume_dist() over (partition by name order by mynumber) as cd,
+ntile(3) over (partition by name order by mynumber) as nt,
+sum(mynumber) over (partition by name order by mynumber) as s
+from vector_ptf_ntile_int;
+
+select name, rowindex, mynumber,
+rank() over (partition by name order by mynumber) as r,
+cume_dist() over (partition by name order by mynumber) as cd,
+ntile(3) over (partition by name order by mynumber) as nt,
+sum(mynumber) over (partition by name order by mynumber) as s
+from vector_ptf_ntile_int;
+
+-- NOT VECTORIZED: a non-constant number of buckets falls back to row mode,
which takes the value
+-- from the first row of each partition
+explain vectorization detail select name, rowindex,
+ntile(rowindex) over (partition by name order by rowindex) as nt
+from vector_ptf_ntile_int;
+
+select name, rowindex,
+ntile(rowindex) over (partition by name order by rowindex) as nt
+from vector_ptf_ntile_int;
+
+-- Default reducer batch size: each partition arrives as a single batch
spanning several buckets
+set hive.vectorized.testing.reducer.batch.size=-1;
+
+set hive.vectorized.execution.ptf.enabled=false;
+
+select name, rowindex, mynumber,
+ntile(10) over (partition by name) as nt
+from vector_ptf_ntile_int;
+
+set hive.vectorized.execution.ptf.enabled=true;
+
+select name, rowindex, mynumber,
+ntile(10) over (partition by name) as nt
+from vector_ptf_ntile_int;
diff --git
a/ql/src/test/results/clientpositive/llap/sketches_materialized_view_ntile.q.out
b/ql/src/test/results/clientpositive/llap/sketches_materialized_view_ntile.q.out
index eed5ef3f6ae..8e3453d70f1 100644
---
a/ql/src/test/results/clientpositive/llap/sketches_materialized_view_ntile.q.out
+++
b/ql/src/test/results/clientpositive/llap/sketches_materialized_view_ntile.q.out
@@ -238,7 +238,7 @@ STAGE PLANS:
Execution mode: vectorized, llap
LLAP IO: may be used (ACID table)
Reducer 2
- Execution mode: llap
+ Execution mode: vectorized, llap
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey1 (type: int)
diff --git
a/ql/src/test/results/clientpositive/llap/sketches_rewrite_ntile.q.out
b/ql/src/test/results/clientpositive/llap/sketches_rewrite_ntile.q.out
index f5b5a34bb95..7ff758223e4 100644
--- a/ql/src/test/results/clientpositive/llap/sketches_rewrite_ntile.q.out
+++ b/ql/src/test/results/clientpositive/llap/sketches_rewrite_ntile.q.out
@@ -606,7 +606,7 @@ STAGE PLANS:
Execution mode: vectorized, llap
LLAP IO: may be used (ACID table)
Reducer 2
- Execution mode: llap
+ Execution mode: vectorized, llap
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey1 (type: char(1))
diff --git
a/ql/src/test/results/clientpositive/llap/vector_ptf_buffered_complex_scratch_col.q.out
b/ql/src/test/results/clientpositive/llap/vector_ptf_buffered_complex_scratch_col.q.out
new file mode 100644
index 00000000000..83e8498481e
--- /dev/null
+++
b/ql/src/test/results/clientpositive/llap/vector_ptf_buffered_complex_scratch_col.q.out
@@ -0,0 +1,291 @@
+PREHOOK: query: DROP TABLE IF EXISTS vector_ptf_complex_scratch
+PREHOOK: type: DROPTABLE
+PREHOOK: Output: database:default
+POSTHOOK: query: DROP TABLE IF EXISTS vector_ptf_complex_scratch
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: database:default
+PREHOOK: query: CREATE TABLE vector_ptf_complex_scratch(name string, rowindex
int, mynumber int) stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@vector_ptf_complex_scratch
+POSTHOOK: query: CREATE TABLE vector_ptf_complex_scratch(name string, rowindex
int, mynumber int) stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@vector_ptf_complex_scratch
+PREHOOK: query: INSERT INTO vector_ptf_complex_scratch values
+('first', 1, 1),
+('first', 2, 1),
+('first', 3, 2),
+('first', 4, 3),
+('first', 5, NULL),
+('second', 6, 10),
+('second', 7, 20),
+('second', 8, 20),
+(NULL, 9, 7)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@vector_ptf_complex_scratch
+POSTHOOK: query: INSERT INTO vector_ptf_complex_scratch values
+('first', 1, 1),
+('first', 2, 1),
+('first', 3, 2),
+('first', 4, 3),
+('first', 5, NULL),
+('second', 6, 10),
+('second', 7, 20),
+('second', 8, 20),
+(NULL, 9, 7)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@vector_ptf_complex_scratch
+POSTHOOK: Lineage: vector_ptf_complex_scratch.mynumber SCRIPT []
+POSTHOOK: Lineage: vector_ptf_complex_scratch.name SCRIPT []
+POSTHOOK: Lineage: vector_ptf_complex_scratch.rowindex SCRIPT []
+PREHOOK: query: select name, rowindex,
+ntile(2) over (partition by name order by rowindex) as nt,
+cume_dist() over (partition by name order by rowindex) as cd,
+sum(mynumber) over (partition by name order by rowindex) as s,
+array(rowindex, mynumber)[1] as arr
+from vector_ptf_complex_scratch
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_complex_scratch
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex,
+ntile(2) over (partition by name order by rowindex) as nt,
+cume_dist() over (partition by name order by rowindex) as cd,
+sum(mynumber) over (partition by name order by rowindex) as s,
+array(rowindex, mynumber)[1] as arr
+from vector_ptf_complex_scratch
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_complex_scratch
+#### A masked pattern was here ####
+NULL 9 1 1.0 7 7
+first 1 1 0.2 1 1
+first 2 1 0.4 2 1
+first 3 1 0.6 4 2
+first 4 2 0.8 7 3
+first 5 2 1.0 7 NULL
+second 6 1 0.3333333333333333 10 10
+second 7 1 0.6666666666666666 30 20
+second 8 2 1.0 50 20
+PREHOOK: query: explain vectorization detail select name, rowindex,
+ntile(2) over (partition by name order by rowindex) as nt,
+cume_dist() over (partition by name order by rowindex) as cd,
+sum(mynumber) over (partition by name order by rowindex) as s,
+array(rowindex, mynumber)[1] as arr
+from vector_ptf_complex_scratch
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_complex_scratch
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization detail select name, rowindex,
+ntile(2) over (partition by name order by rowindex) as nt,
+cume_dist() over (partition by name order by rowindex) as cd,
+sum(mynumber) over (partition by name order by rowindex) as s,
+array(rowindex, mynumber)[1] as arr
+from vector_ptf_complex_scratch
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_complex_scratch
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: vector_ptf_complex_scratch
+ Statistics: Num rows: 9 Data size: 873 Basic stats: COMPLETE
Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:name:string,
1:rowindex:int, 2:mynumber:int,
3:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>,
4:ROW__IS__DELETED:boolean]
+ Reduce Output Operator
+ key expressions: name (type: string), rowindex (type: int)
+ null sort order: az
+ sort order: ++
+ Map-reduce partition columns: name (type: string)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: 0:string, 1:int
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ partitionColumns: 0:string
+ valueColumns: 2:int
+ Statistics: Num rows: 9 Data size: 873 Basic stats:
COMPLETE Column stats: COMPLETE
+ value expressions: mynumber (type: int)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet:
hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats:
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ includeColumns: [0, 1, 2]
+ dataColumns: name:string, rowindex:int, mynumber:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: az
+ reduceColumnSortOrder: ++
+ allNative: false
+ usesVectorUDFAdaptor: true
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ dataColumns: KEY.reducesinkkey0:string,
KEY.reducesinkkey1:int, VALUE._col0:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, double, bigint, bigint,
array<int>, bigint]
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string),
KEY.reducesinkkey1 (type: int), VALUE._col0 (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1, 2]
+ Statistics: Num rows: 9 Data size: 873 Basic stats: COMPLETE
Column stats: COMPLETE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string, _col1: int, _col2: int
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col1 ASC NULLS LAST
+ partition by: _col0
+ raw input shape:
+ window functions:
+ window function definition
+ alias: ntile_window_0
+ arguments: 2
+ name: ntile
+ window function: GenericUDAFNTileEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: cume_dist_window_1
+ arguments: _col1
+ name: cume_dist
+ window function: GenericUDAFCumeDistEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: sum_window_2
+ arguments: _col2
+ name: sum
+ window function: GenericUDAFSumLong
+ window frame: RANGE PRECEDING(MAX)~CURRENT
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorNTile,
VectorPTFEvaluatorCumeDist, VectorPTFEvaluatorLongSum]
+ functionInputExpressions: [ConstantVectorExpression(val
2) -> 6:int, col 1:int, col 2:int]
+ functionNames: [ntile, cume_dist, sum]
+ keyInputColumns: [0, 1]
+ native: true
+ nonKeyInputColumns: [2]
+ orderExpressions: [col 1:int]
+ outputColumns: [3, 4, 5, 0, 1, 2]
+ outputTypes: [int, double, bigint, string, int, int]
+ partitionExpressions: [col 0:string]
+ streamingColumns: [3, 4]
+ Statistics: Num rows: 9 Data size: 873 Basic stats: COMPLETE
Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: int),
ntile_window_0 (type: int), cume_dist_window_1 (type: double), sum_window_2
(type: bigint), array(_col1,_col2)[1] (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1, 3, 4, 5, 8]
+ selectExpressions: ListIndexColScalar(col
7:array<int>, col 1:int)(children: VectorUDFAdaptor(array(_col1,_col2)) ->
7:array<int>) -> 8:int
+ Statistics: Num rows: 9 Data size: 964 Basic stats:
COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 9 Data size: 964 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select name, rowindex,
+ntile(2) over (partition by name order by rowindex) as nt,
+cume_dist() over (partition by name order by rowindex) as cd,
+sum(mynumber) over (partition by name order by rowindex) as s,
+array(rowindex, mynumber)[1] as arr
+from vector_ptf_complex_scratch
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_complex_scratch
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex,
+ntile(2) over (partition by name order by rowindex) as nt,
+cume_dist() over (partition by name order by rowindex) as cd,
+sum(mynumber) over (partition by name order by rowindex) as s,
+array(rowindex, mynumber)[1] as arr
+from vector_ptf_complex_scratch
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_complex_scratch
+#### A masked pattern was here ####
+NULL 9 1 1.0 7 7
+first 1 1 0.2 1 1
+first 2 1 0.4 2 1
+first 3 1 0.6 4 2
+first 4 2 0.8 7 3
+first 5 2 1.0 7 NULL
+second 6 1 0.3333333333333333 10 10
+second 7 1 0.6666666666666666 30 20
+second 8 2 1.0 50 20
+PREHOOK: query: select name, rowindex,
+cume_dist() over (partition by name order by rowindex) as cd,
+array(rowindex, mynumber)[1] as arr
+from vector_ptf_complex_scratch
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_complex_scratch
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex,
+cume_dist() over (partition by name order by rowindex) as cd,
+array(rowindex, mynumber)[1] as arr
+from vector_ptf_complex_scratch
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_complex_scratch
+#### A masked pattern was here ####
+NULL 9 1.0 7
+first 1 0.2 1
+first 2 0.4 1
+first 3 0.6 2
+first 4 0.8 3
+first 5 1.0 NULL
+second 6 0.3333333333333333 10
+second 7 0.6666666666666666 20
+second 8 1.0 20
diff --git a/ql/src/test/results/clientpositive/llap/vector_ptf_ntile.q.out
b/ql/src/test/results/clientpositive/llap/vector_ptf_ntile.q.out
new file mode 100644
index 00000000000..f87e11d6d3b
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/vector_ptf_ntile.q.out
@@ -0,0 +1,1692 @@
+PREHOOK: query: DROP TABLE IF EXISTS vector_ptf_ntile_int
+PREHOOK: type: DROPTABLE
+PREHOOK: Output: database:default
+POSTHOOK: query: DROP TABLE IF EXISTS vector_ptf_ntile_int
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: database:default
+PREHOOK: query: CREATE TABLE vector_ptf_ntile_int(name string, rowindex int,
mynumber int) stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@vector_ptf_ntile_int
+POSTHOOK: query: CREATE TABLE vector_ptf_ntile_int(name string, rowindex int,
mynumber int) stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@vector_ptf_ntile_int
+PREHOOK: query: INSERT INTO vector_ptf_ntile_int values
+
+('first', 1, 1),
+('first', 2, 1),
+('first', 3, 1),
+('first', 4, 2),
+('first', 5, 3),
+('first', 6, 3),
+('first', 7, 4),
+('first', 8, 4),
+('first', 9, 5),
+('first', 10, 5),
+('first', 11, NULL),
+('first', 12, NULL),
+
+('second', 22, 10),
+('second', 23, 10),
+('second', 24, 20),
+('second', 25, 20),
+('second', 26, 30),
+('second', 27, 40),
+('second', 28, 40),
+('second', 29, 50),
+('second', 30, NULL),
+('second', 31, NULL),
+
+(NULL, 43, 7),
+(NULL, 44, 7)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@vector_ptf_ntile_int
+POSTHOOK: query: INSERT INTO vector_ptf_ntile_int values
+
+('first', 1, 1),
+('first', 2, 1),
+('first', 3, 1),
+('first', 4, 2),
+('first', 5, 3),
+('first', 6, 3),
+('first', 7, 4),
+('first', 8, 4),
+('first', 9, 5),
+('first', 10, 5),
+('first', 11, NULL),
+('first', 12, NULL),
+
+('second', 22, 10),
+('second', 23, 10),
+('second', 24, 20),
+('second', 25, 20),
+('second', 26, 30),
+('second', 27, 40),
+('second', 28, 40),
+('second', 29, 50),
+('second', 30, NULL),
+('second', 31, NULL),
+
+(NULL, 43, 7),
+(NULL, 44, 7)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@vector_ptf_ntile_int
+POSTHOOK: Lineage: vector_ptf_ntile_int.mynumber SCRIPT []
+POSTHOOK: Lineage: vector_ptf_ntile_int.name SCRIPT []
+POSTHOOK: Lineage: vector_ptf_ntile_int.rowindex SCRIPT []
+PREHOOK: query: select name, rowindex, mynumber,
+ntile(4) over (partition by name order by mynumber) as nt
+from vector_ptf_ntile_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+ntile(4) over (partition by name order by mynumber) as nt
+from vector_ptf_ntile_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+NULL 44 7 1
+NULL 43 7 2
+first 1 1 1
+first 2 1 1
+first 3 1 1
+first 4 2 2
+first 6 3 2
+first 5 3 2
+first 8 4 3
+first 7 4 3
+first 9 5 3
+first 10 5 4
+first 11 NULL 4
+first 12 NULL 4
+second 22 10 1
+second 23 10 1
+second 25 20 1
+second 24 20 2
+second 26 30 2
+second 27 40 2
+second 28 40 3
+second 29 50 3
+second 30 NULL 4
+second 31 NULL 4
+PREHOOK: query: select name, rowindex, mynumber,
+ntile(3) over (partition by name order by mynumber) as nt3,
+ntile(10) over (partition by name order by mynumber) as nt10
+from vector_ptf_ntile_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+ntile(3) over (partition by name order by mynumber) as nt3,
+ntile(10) over (partition by name order by mynumber) as nt10
+from vector_ptf_ntile_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+NULL 44 7 1 1
+NULL 43 7 2 2
+first 1 1 1 1
+first 2 1 1 1
+first 3 1 1 2
+first 4 2 1 2
+first 6 3 2 3
+first 5 3 2 4
+first 8 4 2 5
+first 7 4 2 6
+first 9 5 3 7
+first 10 5 3 8
+first 11 NULL 3 9
+first 12 NULL 3 10
+second 22 10 1 1
+second 23 10 1 2
+second 25 20 1 3
+second 24 20 1 4
+second 26 30 2 5
+second 27 40 2 6
+second 28 40 2 7
+second 29 50 3 8
+second 30 NULL 3 9
+second 31 NULL 3 10
+PREHOOK: query: select name, rowindex, mynumber,
+ntile(5) over (order by mynumber) as nt
+from vector_ptf_ntile_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+ntile(5) over (order by mynumber) as nt
+from vector_ptf_ntile_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+first 1 1 1
+first 2 1 1
+first 3 1 1
+first 4 2 1
+first 5 3 1
+first 6 3 2
+first 7 4 2
+first 8 4 2
+first 9 5 2
+first 10 5 2
+NULL 44 7 3
+NULL 43 7 3
+second 23 10 3
+second 22 10 3
+second 24 20 3
+second 25 20 4
+second 26 30 4
+second 27 40 4
+second 28 40 4
+second 29 50 4
+first 11 NULL 5
+second 30 NULL 5
+second 31 NULL 5
+first 12 NULL 5
+PREHOOK: query: select name, rowindex, mynumber,
+ntile(4) over (partition by name) as nt
+from vector_ptf_ntile_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+ntile(4) over (partition by name) as nt
+from vector_ptf_ntile_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+NULL 44 7 1
+NULL 43 7 2
+first 1 1 1
+first 2 1 1
+first 3 1 1
+first 4 2 2
+first 5 3 2
+first 6 3 2
+first 7 4 3
+first 8 4 3
+first 9 5 3
+first 10 5 4
+first 11 NULL 4
+first 12 NULL 4
+second 24 20 1
+second 25 20 1
+second 26 30 1
+second 27 40 2
+second 28 40 2
+second 29 50 2
+second 30 NULL 3
+second 31 NULL 3
+second 23 10 4
+second 22 10 4
+PREHOOK: query: select name, rowindex, mynumber,
+ntile(4) over () as nt
+from vector_ptf_ntile_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+ntile(4) over () as nt
+from vector_ptf_ntile_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+first 1 1 1
+first 2 1 1
+first 3 1 1
+first 4 2 1
+first 5 3 1
+first 6 3 1
+first 7 4 2
+first 8 4 2
+first 9 5 2
+first 10 5 2
+first 11 NULL 2
+first 12 NULL 2
+second 22 10 3
+second 23 10 3
+second 24 20 3
+second 25 20 3
+second 26 30 3
+second 27 40 3
+second 28 40 4
+second 29 50 4
+second 30 NULL 4
+second 31 NULL 4
+NULL 43 7 4
+NULL 44 7 4
+PREHOOK: query: select name, rowindex, mynumber,
+rank() over (partition by name order by mynumber) as r,
+cume_dist() over (partition by name order by mynumber) as cd,
+ntile(3) over (partition by name order by mynumber) as nt,
+sum(mynumber) over (partition by name order by mynumber) as s
+from vector_ptf_ntile_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+rank() over (partition by name order by mynumber) as r,
+cume_dist() over (partition by name order by mynumber) as cd,
+ntile(3) over (partition by name order by mynumber) as nt,
+sum(mynumber) over (partition by name order by mynumber) as s
+from vector_ptf_ntile_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+NULL 44 7 1 1.0 1 14
+NULL 43 7 1 1.0 2 14
+first 1 1 1 0.25 1 3
+first 2 1 1 0.25 1 3
+first 3 1 1 0.25 1 3
+first 4 2 4 0.3333333333333333 1 5
+first 6 3 5 0.5 2 11
+first 5 3 5 0.5 2 11
+first 8 4 7 0.6666666666666666 2 19
+first 7 4 7 0.6666666666666666 2 19
+first 9 5 9 0.8333333333333334 3 29
+first 10 5 9 0.8333333333333334 3 29
+first 11 NULL 11 1.0 3 29
+first 12 NULL 11 1.0 3 29
+second 22 10 1 0.2 1 20
+second 23 10 1 0.2 1 20
+second 25 20 3 0.4 1 60
+second 24 20 3 0.4 1 60
+second 26 30 5 0.5 2 90
+second 27 40 6 0.7 2 170
+second 28 40 6 0.7 2 170
+second 29 50 8 0.8 3 220
+second 30 NULL 9 1.0 3 220
+second 31 NULL 9 1.0 3 220
+PREHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+ntile(4) over (partition by name order by mynumber) as nt
+from vector_ptf_ntile_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+ntile(4) over (partition by name order by mynumber) as nt
+from vector_ptf_ntile_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: vector_ptf_ntile_int
+ Statistics: Num rows: 24 Data size: 2227 Basic stats:
COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:name:string,
1:rowindex:int, 2:mynumber:int,
3:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>,
4:ROW__IS__DELETED:boolean]
+ Reduce Output Operator
+ key expressions: name (type: string), mynumber (type: int)
+ null sort order: az
+ sort order: ++
+ Map-reduce partition columns: name (type: string)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: 0:string, 2:int
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ partitionColumns: 0:string
+ valueColumns: 1:int
+ Statistics: Num rows: 24 Data size: 2227 Basic stats:
COMPLETE Column stats: COMPLETE
+ value expressions: rowindex (type: int)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet:
hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats:
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ includeColumns: [0, 1, 2]
+ dataColumns: name:string, rowindex:int, mynumber:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: az
+ reduceColumnSortOrder: ++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ dataColumns: KEY.reducesinkkey0:string,
KEY.reducesinkkey1:int, VALUE._col0:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, bigint]
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string), VALUE._col0
(type: int), KEY.reducesinkkey1 (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 2, 1]
+ Statistics: Num rows: 24 Data size: 2227 Basic stats: COMPLETE
Column stats: COMPLETE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string, _col1: int, _col2: int
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col2 ASC NULLS LAST
+ partition by: _col0
+ raw input shape:
+ window functions:
+ window function definition
+ alias: ntile_window_0
+ arguments: 4
+ name: ntile
+ window function: GenericUDAFNTileEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorNTile]
+ functionInputExpressions: [ConstantVectorExpression(val
4) -> 4:int]
+ functionNames: [ntile]
+ keyInputColumns: [0, 1]
+ native: true
+ nonKeyInputColumns: [2]
+ orderExpressions: [col 1:int]
+ outputColumns: [3, 0, 2, 1]
+ outputTypes: [int, string, int, int]
+ partitionExpressions: [col 0:string]
+ streamingColumns: [3]
+ Statistics: Num rows: 24 Data size: 2227 Basic stats:
COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: int),
_col2 (type: int), ntile_window_0 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 2, 1, 3]
+ Statistics: Num rows: 24 Data size: 2129 Basic stats:
COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 24 Data size: 2129 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select name, rowindex, mynumber,
+ntile(4) over (partition by name order by mynumber) as nt
+from vector_ptf_ntile_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+ntile(4) over (partition by name order by mynumber) as nt
+from vector_ptf_ntile_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+NULL 44 7 1
+NULL 43 7 2
+first 1 1 1
+first 2 1 1
+first 3 1 1
+first 4 2 2
+first 6 3 2
+first 5 3 2
+first 8 4 3
+first 7 4 3
+first 9 5 3
+first 10 5 4
+first 11 NULL 4
+first 12 NULL 4
+second 22 10 1
+second 23 10 1
+second 25 20 1
+second 24 20 2
+second 26 30 2
+second 27 40 2
+second 28 40 3
+second 29 50 3
+second 30 NULL 4
+second 31 NULL 4
+PREHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+ntile(3) over (partition by name order by mynumber) as nt3,
+ntile(10) over (partition by name order by mynumber) as nt10
+from vector_ptf_ntile_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+ntile(3) over (partition by name order by mynumber) as nt3,
+ntile(10) over (partition by name order by mynumber) as nt10
+from vector_ptf_ntile_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: vector_ptf_ntile_int
+ Statistics: Num rows: 24 Data size: 2227 Basic stats:
COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:name:string,
1:rowindex:int, 2:mynumber:int,
3:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>,
4:ROW__IS__DELETED:boolean]
+ Reduce Output Operator
+ key expressions: name (type: string), mynumber (type: int)
+ null sort order: az
+ sort order: ++
+ Map-reduce partition columns: name (type: string)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: 0:string, 2:int
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ partitionColumns: 0:string
+ valueColumns: 1:int
+ Statistics: Num rows: 24 Data size: 2227 Basic stats:
COMPLETE Column stats: COMPLETE
+ value expressions: rowindex (type: int)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet:
hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats:
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ includeColumns: [0, 1, 2]
+ dataColumns: name:string, rowindex:int, mynumber:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: az
+ reduceColumnSortOrder: ++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ dataColumns: KEY.reducesinkkey0:string,
KEY.reducesinkkey1:int, VALUE._col0:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, bigint, bigint, bigint]
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string), VALUE._col0
(type: int), KEY.reducesinkkey1 (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 2, 1]
+ Statistics: Num rows: 24 Data size: 2227 Basic stats: COMPLETE
Column stats: COMPLETE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string, _col1: int, _col2: int
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col2 ASC NULLS LAST
+ partition by: _col0
+ raw input shape:
+ window functions:
+ window function definition
+ alias: ntile_window_0
+ arguments: 3
+ name: ntile
+ window function: GenericUDAFNTileEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: ntile_window_1
+ arguments: 10
+ name: ntile
+ window function: GenericUDAFNTileEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorNTile,
VectorPTFEvaluatorNTile]
+ functionInputExpressions: [ConstantVectorExpression(val
3) -> 5:int, ConstantVectorExpression(val 10) -> 6:int]
+ functionNames: [ntile, ntile]
+ keyInputColumns: [0, 1]
+ native: true
+ nonKeyInputColumns: [2]
+ orderExpressions: [col 1:int]
+ outputColumns: [3, 4, 0, 2, 1]
+ outputTypes: [int, int, string, int, int]
+ partitionExpressions: [col 0:string]
+ streamingColumns: [3, 4]
+ Statistics: Num rows: 24 Data size: 2227 Basic stats:
COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: int),
_col2 (type: int), ntile_window_0 (type: int), ntile_window_1 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 2, 1, 3, 4]
+ Statistics: Num rows: 24 Data size: 2225 Basic stats:
COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 24 Data size: 2225 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select name, rowindex, mynumber,
+ntile(3) over (partition by name order by mynumber) as nt3,
+ntile(10) over (partition by name order by mynumber) as nt10
+from vector_ptf_ntile_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+ntile(3) over (partition by name order by mynumber) as nt3,
+ntile(10) over (partition by name order by mynumber) as nt10
+from vector_ptf_ntile_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+NULL 44 7 1 1
+NULL 43 7 2 2
+first 1 1 1 1
+first 2 1 1 1
+first 3 1 1 2
+first 4 2 1 2
+first 6 3 2 3
+first 5 3 2 4
+first 8 4 2 5
+first 7 4 2 6
+first 9 5 3 7
+first 10 5 3 8
+first 11 NULL 3 9
+first 12 NULL 3 10
+second 22 10 1 1
+second 23 10 1 2
+second 25 20 1 3
+second 24 20 1 4
+second 26 30 2 5
+second 27 40 2 6
+second 28 40 2 7
+second 29 50 3 8
+second 30 NULL 3 9
+second 31 NULL 3 10
+PREHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+ntile(5) over (order by mynumber) as nt
+from vector_ptf_ntile_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+ntile(5) over (order by mynumber) as nt
+from vector_ptf_ntile_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: vector_ptf_ntile_int
+ Statistics: Num rows: 24 Data size: 2227 Basic stats:
COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:name:string,
1:rowindex:int, 2:mynumber:int,
3:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>,
4:ROW__IS__DELETED:boolean]
+ Reduce Output Operator
+ key expressions: 0 (type: int), mynumber (type: int)
+ null sort order: az
+ sort order: ++
+ Map-reduce partition columns: 0 (type: int)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: 5:int, 2:int
+ keyExpressions: ConstantVectorExpression(val 0) ->
5:int
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ partitionColumns: 6:int
+ valueColumns: 0:string, 1:int
+ Statistics: Num rows: 24 Data size: 2227 Basic stats:
COMPLETE Column stats: COMPLETE
+ value expressions: name (type: string), rowindex (type:
int)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet:
hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats:
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ includeColumns: [0, 1, 2]
+ dataColumns: name:string, rowindex:int, mynumber:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, bigint]
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: az
+ reduceColumnSortOrder: ++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 4
+ dataColumns: KEY.reducesinkkey0:int,
KEY.reducesinkkey1:int, VALUE._col0:string, VALUE._col1:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, bigint, bigint]
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string), VALUE._col1 (type:
int), KEY.reducesinkkey1 (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [2, 3, 1]
+ Statistics: Num rows: 24 Data size: 2227 Basic stats: COMPLETE
Column stats: COMPLETE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string, _col1: int, _col2: int
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col2 ASC NULLS LAST
+ partition by: 0
+ raw input shape:
+ window functions:
+ window function definition
+ alias: ntile_window_0
+ arguments: 5
+ name: ntile
+ window function: GenericUDAFNTileEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorNTile]
+ functionInputExpressions: [ConstantVectorExpression(val
5) -> 6:int]
+ functionNames: [ntile]
+ keyInputColumns: [1]
+ native: true
+ nonKeyInputColumns: [2, 3]
+ orderExpressions: [col 1:int]
+ outputColumns: [4, 2, 3, 1]
+ outputTypes: [int, string, int, int]
+ partitionExpressions: [ConstantVectorExpression(val 0)
-> 5:int]
+ streamingColumns: [4]
+ Statistics: Num rows: 24 Data size: 2227 Basic stats:
COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: int),
_col2 (type: int), ntile_window_0 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [2, 3, 1, 4]
+ Statistics: Num rows: 24 Data size: 2129 Basic stats:
COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 24 Data size: 2129 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select name, rowindex, mynumber,
+ntile(5) over (order by mynumber) as nt
+from vector_ptf_ntile_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+ntile(5) over (order by mynumber) as nt
+from vector_ptf_ntile_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+first 1 1 1
+first 2 1 1
+first 3 1 1
+first 4 2 1
+first 5 3 1
+first 6 3 2
+first 7 4 2
+first 8 4 2
+first 9 5 2
+first 10 5 2
+NULL 44 7 3
+NULL 43 7 3
+second 23 10 3
+second 22 10 3
+second 24 20 3
+second 25 20 4
+second 26 30 4
+second 27 40 4
+second 28 40 4
+second 29 50 4
+first 11 NULL 5
+second 30 NULL 5
+second 31 NULL 5
+first 12 NULL 5
+PREHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+ntile(4) over (partition by name) as nt
+from vector_ptf_ntile_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+ntile(4) over (partition by name) as nt
+from vector_ptf_ntile_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: vector_ptf_ntile_int
+ Statistics: Num rows: 24 Data size: 2227 Basic stats:
COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:name:string,
1:rowindex:int, 2:mynumber:int,
3:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>,
4:ROW__IS__DELETED:boolean]
+ Reduce Output Operator
+ key expressions: name (type: string)
+ null sort order: a
+ sort order: +
+ Map-reduce partition columns: name (type: string)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkStringOperator
+ keyColumns: 0:string
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 1:int, 2:int
+ Statistics: Num rows: 24 Data size: 2227 Basic stats:
COMPLETE Column stats: COMPLETE
+ value expressions: rowindex (type: int), mynumber (type:
int)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet:
hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats:
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ includeColumns: [0, 1, 2]
+ dataColumns: name:string, rowindex:int, mynumber:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: a
+ reduceColumnSortOrder: +
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ dataColumns: KEY.reducesinkkey0:string, VALUE._col0:int,
VALUE._col1:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, bigint]
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string), VALUE._col0
(type: int), VALUE._col1 (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1, 2]
+ Statistics: Num rows: 24 Data size: 2227 Basic stats: COMPLETE
Column stats: COMPLETE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string, _col1: int, _col2: int
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col0 ASC NULLS FIRST
+ partition by: _col0
+ raw input shape:
+ window functions:
+ window function definition
+ alias: ntile_window_0
+ arguments: 4
+ name: ntile
+ window function: GenericUDAFNTileEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorNTile]
+ functionInputExpressions: [ConstantVectorExpression(val
4) -> 4:int]
+ functionNames: [ntile]
+ keyInputColumns: [0]
+ native: true
+ nonKeyInputColumns: [1, 2]
+ orderExpressions: [col 0:string]
+ outputColumns: [3, 0, 1, 2]
+ outputTypes: [int, string, int, int]
+ partitionExpressions: [col 0:string]
+ streamingColumns: [3]
+ Statistics: Num rows: 24 Data size: 2227 Basic stats:
COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: int),
_col2 (type: int), ntile_window_0 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1, 2, 3]
+ Statistics: Num rows: 24 Data size: 2129 Basic stats:
COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 24 Data size: 2129 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select name, rowindex, mynumber,
+ntile(4) over (partition by name) as nt
+from vector_ptf_ntile_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+ntile(4) over (partition by name) as nt
+from vector_ptf_ntile_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+NULL 44 7 1
+NULL 43 7 2
+first 1 1 1
+first 2 1 1
+first 3 1 1
+first 4 2 2
+first 5 3 2
+first 6 3 2
+first 7 4 3
+first 8 4 3
+first 9 5 3
+first 10 5 4
+first 11 NULL 4
+first 12 NULL 4
+second 24 20 1
+second 25 20 1
+second 26 30 1
+second 27 40 2
+second 28 40 2
+second 29 50 2
+second 30 NULL 3
+second 31 NULL 3
+second 23 10 4
+second 22 10 4
+PREHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+ntile(4) over () as nt
+from vector_ptf_ntile_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+ntile(4) over () as nt
+from vector_ptf_ntile_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: vector_ptf_ntile_int
+ Statistics: Num rows: 24 Data size: 2227 Basic stats:
COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:name:string,
1:rowindex:int, 2:mynumber:int,
3:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>,
4:ROW__IS__DELETED:boolean]
+ Reduce Output Operator
+ key expressions: 0 (type: int)
+ null sort order: a
+ sort order: +
+ Map-reduce partition columns: 0 (type: int)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkLongOperator
+ keyColumns: 5:int
+ keyExpressions: ConstantVectorExpression(val 0) ->
5:int
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 0:string, 1:int, 2:int
+ Statistics: Num rows: 24 Data size: 2227 Basic stats:
COMPLETE Column stats: COMPLETE
+ value expressions: name (type: string), rowindex (type:
int), mynumber (type: int)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet:
hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats:
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ includeColumns: [0, 1, 2]
+ dataColumns: name:string, rowindex:int, mynumber:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint]
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: a
+ reduceColumnSortOrder: +
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 4
+ dataColumns: KEY.reducesinkkey0:int, VALUE._col0:string,
VALUE._col1:int, VALUE._col2:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, bigint, bigint, bigint]
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string), VALUE._col1 (type:
int), VALUE._col2 (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [1, 2, 3]
+ Statistics: Num rows: 24 Data size: 2227 Basic stats: COMPLETE
Column stats: COMPLETE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string, _col1: int, _col2: int
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: 0 ASC NULLS FIRST
+ partition by: 0
+ raw input shape:
+ window functions:
+ window function definition
+ alias: ntile_window_0
+ arguments: 4
+ name: ntile
+ window function: GenericUDAFNTileEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorNTile]
+ functionInputExpressions: [ConstantVectorExpression(val
4) -> 7:int]
+ functionNames: [ntile]
+ keyInputColumns: []
+ native: true
+ nonKeyInputColumns: [1, 2, 3]
+ orderExpressions: [ConstantVectorExpression(val 0) ->
6:int]
+ outputColumns: [4, 1, 2, 3]
+ outputTypes: [int, string, int, int]
+ partitionExpressions: [ConstantVectorExpression(val 0)
-> 5:int]
+ streamingColumns: [4]
+ Statistics: Num rows: 24 Data size: 2227 Basic stats:
COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: int),
_col2 (type: int), ntile_window_0 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [1, 2, 3, 4]
+ Statistics: Num rows: 24 Data size: 2129 Basic stats:
COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 24 Data size: 2129 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select name, rowindex, mynumber,
+ntile(4) over () as nt
+from vector_ptf_ntile_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+ntile(4) over () as nt
+from vector_ptf_ntile_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+first 1 1 1
+first 2 1 1
+first 3 1 1
+first 4 2 1
+first 5 3 1
+first 6 3 1
+first 7 4 2
+first 8 4 2
+first 9 5 2
+first 10 5 2
+first 11 NULL 2
+first 12 NULL 2
+second 22 10 3
+second 23 10 3
+second 24 20 3
+second 25 20 3
+second 26 30 3
+second 27 40 3
+second 28 40 4
+second 29 50 4
+second 30 NULL 4
+second 31 NULL 4
+NULL 43 7 4
+NULL 44 7 4
+PREHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+rank() over (partition by name order by mynumber) as r,
+cume_dist() over (partition by name order by mynumber) as cd,
+ntile(3) over (partition by name order by mynumber) as nt,
+sum(mynumber) over (partition by name order by mynumber) as s
+from vector_ptf_ntile_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+rank() over (partition by name order by mynumber) as r,
+cume_dist() over (partition by name order by mynumber) as cd,
+ntile(3) over (partition by name order by mynumber) as nt,
+sum(mynumber) over (partition by name order by mynumber) as s
+from vector_ptf_ntile_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: vector_ptf_ntile_int
+ Statistics: Num rows: 24 Data size: 2227 Basic stats:
COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:name:string,
1:rowindex:int, 2:mynumber:int,
3:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>,
4:ROW__IS__DELETED:boolean]
+ Reduce Output Operator
+ key expressions: name (type: string), mynumber (type: int)
+ null sort order: az
+ sort order: ++
+ Map-reduce partition columns: name (type: string)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: 0:string, 2:int
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ partitionColumns: 0:string
+ valueColumns: 1:int
+ Statistics: Num rows: 24 Data size: 2227 Basic stats:
COMPLETE Column stats: COMPLETE
+ value expressions: rowindex (type: int)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet:
hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats:
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ includeColumns: [0, 1, 2]
+ dataColumns: name:string, rowindex:int, mynumber:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: az
+ reduceColumnSortOrder: ++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ dataColumns: KEY.reducesinkkey0:string,
KEY.reducesinkkey1:int, VALUE._col0:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, double, bigint, bigint,
bigint]
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string), VALUE._col0
(type: int), KEY.reducesinkkey1 (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 2, 1]
+ Statistics: Num rows: 24 Data size: 2227 Basic stats: COMPLETE
Column stats: COMPLETE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string, _col1: int, _col2: int
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col2 ASC NULLS LAST
+ partition by: _col0
+ raw input shape:
+ window functions:
+ window function definition
+ alias: rank_window_0
+ arguments: _col2
+ name: rank
+ window function: GenericUDAFRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: cume_dist_window_1
+ arguments: _col2
+ name: cume_dist
+ window function: GenericUDAFCumeDistEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: ntile_window_2
+ arguments: 3
+ name: ntile
+ window function: GenericUDAFNTileEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: sum_window_3
+ arguments: _col2
+ name: sum
+ window function: GenericUDAFSumLong
+ window frame: RANGE PRECEDING(MAX)~CURRENT
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorRank,
VectorPTFEvaluatorCumeDist, VectorPTFEvaluatorNTile, VectorPTFEvaluatorLongSum]
+ functionInputExpressions: [col 1:int, col 1:int,
ConstantVectorExpression(val 3) -> 7:int, col 1:int]
+ functionNames: [rank, cume_dist, ntile, sum]
+ keyInputColumns: [0, 1]
+ native: true
+ nonKeyInputColumns: [2]
+ orderExpressions: [col 1:int]
+ outputColumns: [3, 4, 5, 6, 0, 2, 1]
+ outputTypes: [int, double, int, bigint, string, int, int]
+ partitionExpressions: [col 0:string]
+ streamingColumns: [3, 4, 5]
+ Statistics: Num rows: 24 Data size: 2227 Basic stats:
COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: int),
_col2 (type: int), rank_window_0 (type: int), cume_dist_window_1 (type:
double), ntile_window_2 (type: int), sum_window_3 (type: bigint)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4,
_col5, _col6
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 2, 1, 3, 4, 5, 6]
+ Statistics: Num rows: 24 Data size: 2609 Basic stats:
COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 24 Data size: 2609 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select name, rowindex, mynumber,
+rank() over (partition by name order by mynumber) as r,
+cume_dist() over (partition by name order by mynumber) as cd,
+ntile(3) over (partition by name order by mynumber) as nt,
+sum(mynumber) over (partition by name order by mynumber) as s
+from vector_ptf_ntile_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+rank() over (partition by name order by mynumber) as r,
+cume_dist() over (partition by name order by mynumber) as cd,
+ntile(3) over (partition by name order by mynumber) as nt,
+sum(mynumber) over (partition by name order by mynumber) as s
+from vector_ptf_ntile_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+NULL 44 7 1 1.0 1 14
+NULL 43 7 1 1.0 2 14
+first 1 1 1 0.25 1 3
+first 2 1 1 0.25 1 3
+first 3 1 1 0.25 1 3
+first 4 2 4 0.3333333333333333 1 5
+first 6 3 5 0.5 2 11
+first 5 3 5 0.5 2 11
+first 8 4 7 0.6666666666666666 2 19
+first 7 4 7 0.6666666666666666 2 19
+first 9 5 9 0.8333333333333334 3 29
+first 10 5 9 0.8333333333333334 3 29
+first 11 NULL 11 1.0 3 29
+first 12 NULL 11 1.0 3 29
+second 22 10 1 0.2 1 20
+second 23 10 1 0.2 1 20
+second 25 20 3 0.4 1 60
+second 24 20 3 0.4 1 60
+second 26 30 5 0.5 2 90
+second 27 40 6 0.7 2 170
+second 28 40 6 0.7 2 170
+second 29 50 8 0.8 3 220
+second 30 NULL 9 1.0 3 220
+second 31 NULL 9 1.0 3 220
+PREHOOK: query: explain vectorization detail select name, rowindex,
+ntile(rowindex) over (partition by name order by rowindex) as nt
+from vector_ptf_ntile_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization detail select name, rowindex,
+ntile(rowindex) over (partition by name order by rowindex) as nt
+from vector_ptf_ntile_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: vector_ptf_ntile_int
+ Statistics: Num rows: 24 Data size: 2143 Basic stats:
COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:name:string,
1:rowindex:int, 2:mynumber:int,
3:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>,
4:ROW__IS__DELETED:boolean]
+ Reduce Output Operator
+ key expressions: name (type: string), rowindex (type: int)
+ null sort order: az
+ sort order: ++
+ Map-reduce partition columns: name (type: string)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: 0:string, 1:int
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ partitionColumns: 0:string
+ Statistics: Num rows: 24 Data size: 2143 Basic stats:
COMPLETE Column stats: COMPLETE
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet:
hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats:
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ includeColumns: [0, 1]
+ dataColumns: name:string, rowindex:int, mynumber:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ notVectorizedReason: PTF operator: ntile only supports a
positive integer constant number of buckets
+ vectorized: false
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string),
KEY.reducesinkkey1 (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 24 Data size: 2143 Basic stats: COMPLETE
Column stats: COMPLETE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string, _col1: int
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col1 ASC NULLS LAST
+ partition by: _col0
+ raw input shape:
+ window functions:
+ window function definition
+ alias: ntile_window_0
+ arguments: _col1
+ name: ntile
+ window function: GenericUDAFNTileEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ Statistics: Num rows: 24 Data size: 2143 Basic stats:
COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: int),
ntile_window_0 (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 24 Data size: 2061 Basic stats:
COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 24 Data size: 2061 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select name, rowindex,
+ntile(rowindex) over (partition by name order by rowindex) as nt
+from vector_ptf_ntile_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex,
+ntile(rowindex) over (partition by name order by rowindex) as nt
+from vector_ptf_ntile_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+NULL 43 1
+NULL 44 2
+first 1 1
+first 2 1
+first 3 1
+first 4 1
+first 5 1
+first 6 1
+first 7 1
+first 8 1
+first 9 1
+first 10 1
+first 11 1
+first 12 1
+second 22 1
+second 23 2
+second 24 3
+second 25 4
+second 26 5
+second 27 6
+second 28 7
+second 29 8
+second 30 9
+second 31 10
+PREHOOK: query: select name, rowindex, mynumber,
+ntile(10) over (partition by name) as nt
+from vector_ptf_ntile_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+ntile(10) over (partition by name) as nt
+from vector_ptf_ntile_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+NULL 44 7 1
+NULL 43 7 2
+first 1 1 1
+first 2 1 1
+first 3 1 2
+first 4 2 2
+first 5 3 3
+first 6 3 4
+first 7 4 5
+first 8 4 6
+first 9 5 7
+first 10 5 8
+first 11 NULL 9
+first 12 NULL 10
+second 24 20 1
+second 25 20 2
+second 26 30 3
+second 27 40 4
+second 28 40 5
+second 29 50 6
+second 30 NULL 7
+second 31 NULL 8
+second 23 10 9
+second 22 10 10
+PREHOOK: query: select name, rowindex, mynumber,
+ntile(10) over (partition by name) as nt
+from vector_ptf_ntile_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+ntile(10) over (partition by name) as nt
+from vector_ptf_ntile_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_ntile_int
+#### A masked pattern was here ####
+NULL 44 7 1
+NULL 43 7 2
+first 1 1 1
+first 2 1 1
+first 3 1 2
+first 4 2 2
+first 5 3 3
+first 6 3 4
+first 7 4 5
+first 8 4 6
+first 9 5 7
+first 10 5 8
+first 11 NULL 9
+first 12 NULL 10
+second 24 20 1
+second 25 20 2
+second 26 30 3
+second 27 40 4
+second 28 40 5
+second 29 50 6
+second 30 NULL 7
+second 31 NULL 8
+second 23 10 9
+second 22 10 10
diff --git a/ql/src/test/results/clientpositive/llap/vector_windowing.q.out
b/ql/src/test/results/clientpositive/llap/vector_windowing.q.out
index f0e0c408f60..a80f4013ba0 100644
--- a/ql/src/test/results/clientpositive/llap/vector_windowing.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_windowing.q.out
@@ -3250,7 +3250,7 @@ STAGE PLANS:
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
- notVectorizedReason: PTF operator: ntile not in supported
functions [avg, count, cume_dist, dense_rank, first_value, lag, last_value,
lead, max, min, percent_rank, rank, row_number, sum]
+ notVectorizedReason: PTF operator: stddev_pop not in supported
functions [avg, count, cume_dist, dense_rank, first_value, lag, last_value,
lead, max, min, ntile, percent_rank, rank, row_number, sum]
vectorized: false
Reduce Operator Tree:
Select Operator
