This is an automated email from the ASF dual-hosted git repository.
deniskuzZ pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 04e465d64c9 HIVE-30132: LLAP IO: LlapRecordReader recomputes its
missing-column list for every row batch (#6846)
04e465d64c9 is described below
commit 04e465d64c904c4fd8c5c5b209e8bc594a389962
Author: Denys Kuzmenko <[email protected]>
AuthorDate: Sat Oct 10 07:49:50 2026 +0200
HIVE-30132: LLAP IO: LlapRecordReader recomputes its missing-column list
for every row batch (#6846)
---
.../apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java | 11 +++++------
1 file changed, 5 insertions(+), 6 deletions(-)
diff --git
a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java
b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java
index afd39015b4a..43570480159 100644
---
a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java
+++
b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java
@@ -23,7 +23,6 @@
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -84,7 +83,6 @@
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
-import static java.util.stream.Collectors.toList;
import static org.apache.hadoop.hive.llap.LlapHiveUtils.throwIfCacheOnlyRead;
class LlapRecordReader implements RecordReader<NullWritable,
VectorizedRowBatch>, Consumer<ColumnVectorBatch> {
@@ -94,6 +92,7 @@ class LlapRecordReader implements RecordReader<NullWritable,
VectorizedRowBatch>
private final FileSplit split;
private final IncludesImpl includes;
+ private final List<Integer> missingColIndices;
private final SearchArgument sarg;
private final VectorizedRowBatchCtx rbCtx;
private final boolean isVectorized;
@@ -228,6 +227,8 @@ private LlapRecordReader(MapWork mapWork, JobConf job,
FileSplit split,
// Create the consumer of encoded data; it will coordinate decoding to
CVBs.
feedback = rp = cvp.createReadPipeline(this, split, includes, sarg,
counters, includes,
sourceInputFormat, sourceSerDe, reporter, job,
mapWork.getPathToPartitionInfo());
+ missingColIndices = includes.getReaderLogicalColumnIds().stream()
+ .filter(idx ->
!includes.getLogicalOrderedColumnIds().contains(idx)).toList();
}
private static int getQueueVar(ConfVars var, JobConf jobConf, Configuration
daemonConf) {
@@ -445,8 +446,6 @@ public boolean next(NullWritable key, VectorizedRowBatch
vrb) throws IOException
cvb.swapColumnVector(ixInReadSet, vrb.cols, ixInVrb);
}
// null out col vectors for which the (ORC) file had no data
- List<Integer> missingColIndices =
includes.getReaderLogicalColumnIds().stream()
- .filter(idx ->
!includes.getLogicalOrderedColumnIds().contains(idx)).collect(toList());
if (missingColIndices.size() != (cvb.cols.length - cvbColsPresent)) {
throw new RuntimeException("Unexpected number of missing columns,
expected " + missingColIndices.size() +
", but reader returned " + (cvb.cols.length - cvbColsPresent) + "
missing column vectors.");
@@ -791,7 +790,7 @@ private void generateLogicalOrderedColumnIds() {
"orc.force.positional.evolution turned on.");
return;
}
- logicalOrderedColumnIds = new LinkedList<>();
+ logicalOrderedColumnIds = new ArrayList<>();
Map<Integer, String> fileSchemaMap = new HashMap<>();
Map<String, Integer> readSchemaMap = new HashMap<>();
int order = 0;
@@ -881,7 +880,7 @@ public int getProbeColIdx() {
* @param evolution - provided by ORC libs as per file schema and read
schema
*/
private void adjustPhysicalColumnIds(SchemaEvolution evolution) {
- LinkedList<Integer> newFilePhysicalColumnIds = new LinkedList<>();
+ List<Integer> newFilePhysicalColumnIds = new ArrayList<>();
boolean[] firstLevelPhysicalIncludes =
OrcInputFormat.firstLevelFileIncludes(evolution);
for (int i = 1; i < firstLevelPhysicalIncludes.length; ++i) {
if (firstLevelPhysicalIncludes[i]) {
