deniskuzZ commented on code in PR #6512:
URL: https://github.com/apache/hive/pull/6512#discussion_r3558247672
##########
ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java:
##########
@@ -5190,6 +5244,11 @@ private static VectorPTFInfo
createVectorPTFInfo(Operator<? extends OperatorDesc
int[] keyInputColumnMap =
ArrayUtils.toPrimitive(keyInputColumns.toArray(new Integer[0]));
int[] nonKeyInputColumnMap =
ArrayUtils.toPrimitive(nonKeyInputColumns.toArray(new Integer[0]));
+ if (isPartitionOrderBy && partitionKeyCount > 1) {
+ reorderPartitionColumnsToMatchOutputOrder(outputSignature,
evaluatorCount, outputColumnProjectionMap,
Review Comment:
````
Subject: [PATCH] patch
---
Index:
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFOperator.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFOperator.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFOperator.java
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFOperator.java
(revision 1b1fa6d6a00f38ff8968bea673b5006615cbc2b3)
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFOperator.java
(date 1783678424148)
@@ -349,6 +349,7 @@
bufferedTypeInfos,
orderColumnMap,
keyWithoutOrderColumnMap,
+ partitionColumnMap,
overflowBatch);
isFirstPartition = true;
Index:
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFGroupBatches.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFGroupBatches.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFGroupBatches.java
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFGroupBatches.java
(revision 1b1fa6d6a00f38ff8968bea673b5006615cbc2b3)
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFGroupBatches.java
(date 1783678635446)
@@ -26,6 +26,7 @@
import java.util.List;
import java.util.stream.Collectors;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
import org.apache.hadoop.hive.ql.exec.PTFPartition;
@@ -64,6 +65,7 @@
private int[] bufferedColumnMap;
private int[] orderColumnMap;
private int[] keyWithoutOrderColumnMap;
+ private int[] partitionKeyIndices;
private int spillLimitBufferedBatchCount;
private String spillLocalDirs;
@@ -289,13 +291,21 @@
public void init(VectorPTFEvaluatorBase[] evaluators, int[]
outputProjectionColumnMap,
int[] bufferedColumnMap, TypeInfo[] bufferedTypeInfos, int[]
orderColumnMap,
- int[] keyWithoutOrderColumnMap, VectorizedRowBatch overflowBatch) {
+ int[] keyWithoutOrderColumnMap, int[] partitionColumnMap,
VectorizedRowBatch overflowBatch) {
this.evaluators = evaluators;
this.outputProjectionColumnMap = outputProjectionColumnMap;
this.bufferedColumnMap = bufferedColumnMap;
this.orderColumnMap = orderColumnMap;
this.keyWithoutOrderColumnMap = keyWithoutOrderColumnMap;
+ partitionKeyIndices = new int[keyWithoutOrderColumnMap.length];
+ for (int i = 0; i < keyWithoutOrderColumnMap.length; i++) {
+ partitionKeyIndices[i] = ArrayUtils.indexOf(partitionColumnMap,
keyWithoutOrderColumnMap[i]);
+ Preconditions.checkState(partitionKeyIndices[i] !=
ArrayUtils.INDEX_NOT_FOUND,
+ "Key input column %s not found among partition columns %s",
+ keyWithoutOrderColumnMap[i], Arrays.toString(partitionColumnMap));
+ }
+
this.overflowBatch = overflowBatch;
bufferedBatches = new ArrayList<BufferedVectorizedRowBatch>(0);
@@ -775,6 +785,11 @@
}
}
+ /**
+ * Sets the partition key values as repeating columns in the
overflowBatch. Key columns are in
+ * output projection order, partitionKey values in PARTITION BY order;
partitionKeyIndices maps
+ * between the two.
+ */
private void copyPartitionColumnToOverflow(Object[] partitionKey) {
// Set partition column in overflowBatch.
// We can set by ref since our last batch is held by us.
@@ -782,7 +797,7 @@
for (int i = 0; i < keyInputColumnCount; i++) {
final int keyColumnNum = keyWithoutOrderColumnMap[i];
Preconditions.checkState(overflowBatch.cols[keyColumnNum] != null);
- setRepeatingColumn(partitionKey[i], overflowBatch, keyColumnNum);
+ setRepeatingColumn(partitionKey[partitionKeyIndices[i]],
overflowBatch, keyColumnNum);
}
}
Index:
ql/src/test/org/apache/hadoop/hive/ql/exec/vector/ptf/TestVectorPTFGroupBatches.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/ptf/TestVectorPTFGroupBatches.java
b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/ptf/TestVectorPTFGroupBatches.java
---
a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/ptf/TestVectorPTFGroupBatches.java
(revision 1b1fa6d6a00f38ff8968bea673b5006615cbc2b3)
+++
b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/ptf/TestVectorPTFGroupBatches.java
(date 1783678472285)
@@ -588,6 +588,7 @@
/* bufferedTypeInfos */ new TypeInfo[] { getTypeInfo("int"),
getTypeInfo("string") },
/* orderColumnMap */ new int[] { 1 }, // p_date
/* keyWithoutOrderColumnMap */ new int[] { 0 }, // p_mfgr
+ /* partitionColumnMap */ new int[] { 0 }, // p_mfgr
getFakeOperator().setupOverflowBatch(3, new String[] { "bigint",
"bigint" },
outputProjectionColumnMap, outputTypeInfos));
}
````
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
