This is an automated email from the ASF dual-hosted git repository.

deniskuzZ pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new 875607b9ee4 HIVE-30102: Iceberg: LLAP cannot read an ORC file that has 
positional deletes (#6920)
875607b9ee4 is described below

commit 875607b9ee452bbfc4b2fec81ea217c039041434
Author: Denys Kuzmenko <[email protected]>
AuthorDate: Sat Oct 31 12:49:27 2026 +0100

    HIVE-30102: Iceberg: LLAP cannot read an ORC file that has positional 
deletes (#6920)
---
 .../mr/hive/vector/HiveVectorizedReader.java       |   6 +-
 .../iceberg/mr/mapreduce/IcebergRecordReader.java  |   3 +
 .../iceberg/mr/hive/TestHiveIcebergCRUD.java       |  26 ++
 .../hive/vector/TestHiveIcebergVectorization.java  |  25 ++
 .../positive/llap_iceberg_read_orc_deletes.q       |  26 ++
 .../llap/llap_iceberg_read_orc_deletes.q.out       | 138 ++++++++++
 .../test/resources/testconfiguration.properties    |   2 +
 .../hive/llap/io/api/impl/ColumnVectorBatch.java   |  22 +-
 .../hive/llap/io/api/impl/LlapRecordReader.java    |  79 +++---
 .../io/decode/GenericColumnVectorProducer.java     |  12 +
 .../llap/io/decode/OrcEncodedDataConsumer.java     |  41 ++-
 .../llap/io/metadata/ConsumerFileMetadata.java     |   4 +
 .../TestOrcEncodedDataConsumerRowPositions.java    | 299 +++++++++++++++++++++
 13 files changed, 624 insertions(+), 59 deletions(-)

diff --git 
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveVectorizedReader.java
 
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveVectorizedReader.java
index 52140dfb88b..99a609a36a4 100644
--- 
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveVectorizedReader.java
+++ 
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveVectorizedReader.java
@@ -61,7 +61,6 @@
 import org.apache.iceberg.io.CloseableIterable;
 import org.apache.iceberg.io.CloseableIterator;
 import org.apache.iceberg.io.FileIO;
-import org.apache.iceberg.mr.InputFormatConfig;
 import org.apache.iceberg.mr.hive.HiveIcebergInputFormat;
 import org.apache.iceberg.mr.mapred.MapredIcebergInputFormat;
 import org.apache.iceberg.orc.VectorizedReadUtils;
@@ -220,6 +219,8 @@ private static RecordReader<NullWritable, 
VectorizedRowBatch> orcRecordReader(Jo
     // Need to turn positional schema evolution off since we use column name 
based schema evolution for projection
     // and Iceberg will make a mapping between the file schema and the current 
reading schema.
     job.setBoolean(OrcConf.FORCE_POSITIONAL_EVOLUTION.getHiveConfName(), 
false);
+    // Row positions are derived from the batch index, and ORC's row-level 
filter can drop rows and whole batches.
+    job.setBoolean(OrcConf.ALLOW_SARG_TO_FILTER.getHiveConfName(), false);
 
     // Metadata information has to be passed along in the OrcSplit. Without 
specifying this, the vectorized
     // reader will assume that the ORC file ends at the task's start + length, 
and might fail reading the tail..
@@ -230,9 +231,8 @@ private static RecordReader<NullWritable, 
VectorizedRowBatch> orcRecordReader(Jo
         
VectorizedReadUtils.deserializeToShadedOrcTail(serializedOrcTail).getSchema(), 
residual);
 
     // If LLAP enabled, try to retrieve an LLAP record reader - this might 
yield to null in some special cases
-    // TODO: add support for reading files with positional deletes with LLAP 
(LLAP would need to provide file row num)
     if (HiveConf.getBoolVar(job, HiveConf.ConfVars.LLAP_IO_ENABLED, 
LlapProxy.isDaemon()) &&
-        LlapProxy.getIo() != null && task.deletes().isEmpty() && 
!InputFormatConfig.fetchVirtualColumns(job)) {
+        LlapProxy.getIo() != null) {
       recordReader = LlapProxy.getIo().orc().vectorizedReaderForPath(fileId, 
path, null, readColumnIds,
           job, start, length, reporter);
     }
diff --git 
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java
 
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java
index 2ea261f2014..52faaf20230 100644
--- 
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java
+++ 
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java
@@ -34,6 +34,7 @@
 import org.apache.hadoop.hive.ql.io.IOContextAwareRecordReader;
 import org.apache.hadoop.mapreduce.InputSplit;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hive.iceberg.org.apache.orc.OrcConf;
 import org.apache.iceberg.DataFile;
 import org.apache.iceberg.DataTask;
 import org.apache.iceberg.FileFormat;
@@ -262,6 +263,8 @@ private CloseableIterable<T> newOrcIterable(
     ORC.ReadBuilder orcReadBuilder = ORC.read(inputFile)
         .project(readSchemaWithoutConstantAndMetadataFields)
         .filter(residual)
+        // Row positions are derived from the batch index, and ORC's row-level 
filter can drop rows and whole batches.
+        .config(OrcConf.ALLOW_SARG_TO_FILTER.getHiveConfName(), "false")
         .caseSensitive(isCaseSensitive())
         .split(task.start(), task.length());
 
diff --git 
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergCRUD.java
 
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergCRUD.java
index e3236fe2c3e..ee4d285a9ab 100644
--- 
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergCRUD.java
+++ 
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergCRUD.java
@@ -215,6 +215,32 @@ public void 
testReadAndWriteFormatV2Partitioned_PosDelete() throws IOException {
     Assert.assertArrayEquals(new Object[] {2L, "Trudy", "Pink"}, 
objects.get(3));
   }
 
+  /**
+   * ORC can filter rows by the pushed down predicate; in row mode, row 
positions and deletes must still apply to
+   * the right rows.
+   */
+  @Test
+  public void testPositionDeleteWithOrcRowFiltering() {
+    Assume.assumeTrue(fileFormat == FileFormat.ORC && formatVersion >= 2 && 
!isVectorized);
+
+    Schema schema = new Schema(optional(1, "customer_id", 
Types.LongType.get()));
+    List<Record> records = TestHelper.generateRandomRecords(schema, 30000, 0L);
+    for (int i = 0; i < records.size(); ++i) {
+      records.get(i).setField("customer_id", (long) i);
+    }
+    testTables.createTable(shell, "customers", schema, 
PartitionSpec.unpartitioned(), fileFormat, records,
+        formatVersion);
+    shell.executeStatement("DELETE FROM customers WHERE customer_id = 25000");
+
+    shell.setHiveSessionValue("orc.sarg.to.filter", true);
+    shell.setHiveSessionValue("orc.filter.use.selected", true);
+    List<Object[]> result = shell.executeStatement(
+        "SELECT customer_id, ROW__POSITION FROM customers WHERE customer_id IN 
(25000, 25001)");
+
+    Assert.assertEquals(1, result.size());
+    Assert.assertArrayEquals(new Object[] {25001L, 25001L}, result.get(0));
+  }
+
   /**
    * Tests reading/consuming legacy Iceberg V2 tables containing position 
delete files with embedded
    * row data. Iceberg 1.12.0 removed writer support for row data in position 
deletes, but Hive
diff --git 
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/vector/TestHiveIcebergVectorization.java
 
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/vector/TestHiveIcebergVectorization.java
index 28487b715ff..e57ab182367 100644
--- 
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/vector/TestHiveIcebergVectorization.java
+++ 
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/vector/TestHiveIcebergVectorization.java
@@ -219,6 +219,31 @@ private void 
testVectorizedReadWithDeleteFilter(Map<String, String> props) {
     validation.apply(1501);
   }
 
+  /**
+   * ORC can filter the rows of a batch by the pushed down predicate; row 
positions and deletes must still apply to
+   * the right rows.
+   */
+  @Test
+  public void testHiveDeleteFilterWithOrcRowFiltering() {
+    Assume.assumeTrue(fileFormat == FileFormat.ORC);
+
+    Schema schema = new Schema(optional(1, "customer_id", 
Types.LongType.get()));
+    List<Record> records = TestHelper.generateRandomRecords(schema, 30000, 0L);
+    for (int i = 0; i < records.size(); ++i) {
+      records.get(i).setField("customer_id", (long) i);
+    }
+    testTables.createTable(shell, "vectordelete", schema, 
PartitionSpec.unpartitioned(), fileFormat, records, 2);
+    shell.executeStatement("DELETE FROM vectordelete WHERE customer_id = 
25000");
+
+    shell.setHiveSessionValue("orc.sarg.to.filter", true);
+    shell.setHiveSessionValue("orc.filter.use.selected", true);
+    List<Object[]> result = shell.executeStatement(
+        "select customer_id, ROW__POSITION from vectordelete where customer_id 
in (25000, 25001)");
+
+    Assert.assertEquals(1, result.size());
+    Assert.assertArrayEquals(new Object[] { 25001L, 25001L }, result.get(0));
+  }
+
   @Test
   public void testHiveDeleteFilterWithFilteredParquetBlock() {
     Assume.assumeTrue(fileFormat == FileFormat.PARQUET);
diff --git 
a/iceberg/iceberg-handler/src/test/queries/positive/llap_iceberg_read_orc_deletes.q
 
b/iceberg/iceberg-handler/src/test/queries/positive/llap_iceberg_read_orc_deletes.q
new file mode 100644
index 00000000000..bc37f0a23c4
--- /dev/null
+++ 
b/iceberg/iceberg-handler/src/test/queries/positive/llap_iceberg_read_orc_deletes.q
@@ -0,0 +1,26 @@
+-- ORC files with position deletes are read through LLAP IO (see the LLAP IO 
counters): the deletes and ROW__POSITION
+-- apply to the right rows in a later row group and across a batch boundary 
(the third row group starts at file row
+-- 20000, its sixth batch at 25120)
+set hive.llap.io.enabled=true;
+set hive.vectorized.execution.enabled=true;
+set hive.fetch.task.conversion=none;
+set hive.optimize.index.filter=true;
+
+-- V2: position delete files
+CREATE EXTERNAL TABLE ice_orc_v2 (id INT) STORED BY ICEBERG STORED AS ORC
+TBLPROPERTIES ('format-version'='2');
+INSERT INTO ice_orc_v2 SELECT pos FROM (SELECT posexplode(split(space(29999), 
' ')) AS (pos, val)) t;
+DELETE FROM ice_orc_v2 WHERE id IN (25119, 25121);
+
+-- V3: deletion vectors
+CREATE EXTERNAL TABLE ice_orc_v3 (id INT) STORED BY ICEBERG STORED AS ORC
+TBLPROPERTIES ('format-version'='3');
+INSERT INTO ice_orc_v3 SELECT pos FROM (SELECT posexplode(split(space(29999), 
' ')) AS (pos, val)) t;
+DELETE FROM ice_orc_v3 WHERE id IN (25119, 25121);
+
+SET 
hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.PostExecTezSummaryPrinter;
+SELECT id, ROW__POSITION FROM ice_orc_v2 WHERE id BETWEEN 25117 AND 25123 
ORDER BY id;
+SELECT id, ROW__POSITION FROM ice_orc_v3 WHERE id BETWEEN 25117 AND 25123 
ORDER BY id;
+
+DROP TABLE ice_orc_v2;
+DROP TABLE ice_orc_v3;
diff --git 
a/iceberg/iceberg-handler/src/test/results/positive/llap/llap_iceberg_read_orc_deletes.q.out
 
b/iceberg/iceberg-handler/src/test/results/positive/llap/llap_iceberg_read_orc_deletes.q.out
new file mode 100644
index 00000000000..d49ae17cb09
--- /dev/null
+++ 
b/iceberg/iceberg-handler/src/test/results/positive/llap/llap_iceberg_read_orc_deletes.q.out
@@ -0,0 +1,138 @@
+PREHOOK: query: CREATE EXTERNAL TABLE ice_orc_v2 (id INT) STORED BY ICEBERG 
STORED AS ORC
+TBLPROPERTIES ('format-version'='2')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ice_orc_v2
+POSTHOOK: query: CREATE EXTERNAL TABLE ice_orc_v2 (id INT) STORED BY ICEBERG 
STORED AS ORC
+TBLPROPERTIES ('format-version'='2')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ice_orc_v2
+PREHOOK: query: INSERT INTO ice_orc_v2 SELECT pos FROM (SELECT 
posexplode(split(space(29999), ' ')) AS (pos, val)) t
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc_v2
+POSTHOOK: query: INSERT INTO ice_orc_v2 SELECT pos FROM (SELECT 
posexplode(split(space(29999), ' ')) AS (pos, val)) t
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc_v2
+PREHOOK: query: DELETE FROM ice_orc_v2 WHERE id IN (25119, 25121)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_orc_v2
+PREHOOK: Output: default@ice_orc_v2
+POSTHOOK: query: DELETE FROM ice_orc_v2 WHERE id IN (25119, 25121)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_orc_v2
+POSTHOOK: Output: default@ice_orc_v2
+PREHOOK: query: CREATE EXTERNAL TABLE ice_orc_v3 (id INT) STORED BY ICEBERG 
STORED AS ORC
+TBLPROPERTIES ('format-version'='3')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ice_orc_v3
+POSTHOOK: query: CREATE EXTERNAL TABLE ice_orc_v3 (id INT) STORED BY ICEBERG 
STORED AS ORC
+TBLPROPERTIES ('format-version'='3')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ice_orc_v3
+PREHOOK: query: INSERT INTO ice_orc_v3 SELECT pos FROM (SELECT 
posexplode(split(space(29999), ' ')) AS (pos, val)) t
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc_v3
+POSTHOOK: query: INSERT INTO ice_orc_v3 SELECT pos FROM (SELECT 
posexplode(split(space(29999), ' ')) AS (pos, val)) t
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc_v3
+PREHOOK: query: DELETE FROM ice_orc_v3 WHERE id IN (25119, 25121)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_orc_v3
+PREHOOK: Output: default@ice_orc_v3
+POSTHOOK: query: DELETE FROM ice_orc_v3 WHERE id IN (25119, 25121)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_orc_v3
+POSTHOOK: Output: default@ice_orc_v3
+PREHOOK: query: SELECT id, ROW__POSITION FROM ice_orc_v2 WHERE id BETWEEN 
25117 AND 25123 ORDER BY id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_orc_v2
+#### A masked pattern was here ####
+Stage-1 FILE SYSTEM COUNTERS:
+Stage-1 HIVE COUNTERS:
+   CREATED_FILES: 1
+   DESERIALIZE_ERRORS: 0
+   RECORDS_IN_Map_1: 9998
+   RECORDS_OUT_0: 5
+   RECORDS_OUT_INTERMEDIATE_Map_1: 5
+   RECORDS_OUT_INTERMEDIATE_Reducer_2: 0
+   RECORDS_OUT_OPERATOR_FIL_7: 5
+   RECORDS_OUT_OPERATOR_FS_11: 5
+   RECORDS_OUT_OPERATOR_MAP_0: 0
+   RECORDS_OUT_OPERATOR_RS_9: 5
+   RECORDS_OUT_OPERATOR_SEL_10: 5
+   RECORDS_OUT_OPERATOR_SEL_8: 5
+   RECORDS_OUT_OPERATOR_TS_0: 9998
+   TOTAL_TABLE_ROWS_WRITTEN: 0
+Stage-1 LLAP IO COUNTERS:
+   CACHE_HIT_BYTES: 229
+   CACHE_MISS_BYTES: 0
+   METADATA_CACHE_HIT: 2
+   NUM_DECODED_BATCHES: 1
+   NUM_VECTOR_BATCHES: 10
+   ROWS_EMITTED: 10000
+   SELECTED_ROWGROUPS: 1
+Stage-1 INPUT COUNTERS:
+   GROUPED_INPUT_SPLITS_Map_1: 1
+   INPUT_DIRECTORIES_Map_1: 1
+   INPUT_FILES_Map_1: 1
+   RAW_INPUT_SPLITS_Map_1: 1
+25117  25117
+25118  25118
+25120  25120
+25122  25122
+25123  25123
+PREHOOK: query: SELECT id, ROW__POSITION FROM ice_orc_v3 WHERE id BETWEEN 
25117 AND 25123 ORDER BY id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_orc_v3
+#### A masked pattern was here ####
+Stage-1 FILE SYSTEM COUNTERS:
+Stage-1 HIVE COUNTERS:
+   CREATED_FILES: 1
+   DESERIALIZE_ERRORS: 0
+   RECORDS_IN_Map_1: 9998
+   RECORDS_OUT_0: 5
+   RECORDS_OUT_INTERMEDIATE_Map_1: 5
+   RECORDS_OUT_INTERMEDIATE_Reducer_2: 0
+   RECORDS_OUT_OPERATOR_FIL_7: 5
+   RECORDS_OUT_OPERATOR_FS_11: 5
+   RECORDS_OUT_OPERATOR_MAP_0: 0
+   RECORDS_OUT_OPERATOR_RS_9: 5
+   RECORDS_OUT_OPERATOR_SEL_10: 5
+   RECORDS_OUT_OPERATOR_SEL_8: 5
+   RECORDS_OUT_OPERATOR_TS_0: 9998
+   TOTAL_TABLE_ROWS_WRITTEN: 0
+Stage-1 LLAP IO COUNTERS:
+   CACHE_HIT_BYTES: 229
+   CACHE_MISS_BYTES: 0
+   METADATA_CACHE_HIT: 2
+   NUM_DECODED_BATCHES: 1
+   NUM_VECTOR_BATCHES: 10
+   ROWS_EMITTED: 10000
+   SELECTED_ROWGROUPS: 1
+Stage-1 INPUT COUNTERS:
+   GROUPED_INPUT_SPLITS_Map_1: 1
+   INPUT_DIRECTORIES_Map_1: 1
+   INPUT_FILES_Map_1: 1
+   RAW_INPUT_SPLITS_Map_1: 1
+25117  25117
+25118  25118
+25120  25120
+25122  25122
+25123  25123
+PREHOOK: query: DROP TABLE ice_orc_v2
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@ice_orc_v2
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ice_orc_v2
+PREHOOK: query: DROP TABLE ice_orc_v3
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@ice_orc_v3
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ice_orc_v3
diff --git a/itests/src/test/resources/testconfiguration.properties 
b/itests/src/test/resources/testconfiguration.properties
index e375f4f16e7..8de15596c5d 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -423,6 +423,7 @@ iceberg.llap.query.files=\
   iceberg_merge_files.q,\
   iceberg_row_mode_virtual_columns.q,\
   llap_iceberg_read_orc.q,\
+  llap_iceberg_read_orc_deletes.q,\
   llap_iceberg_read_parquet.q,\
   puffin_col_stats_with_time_travel.q,\
   vectorized_iceberg_read_mixed.q,\
@@ -483,6 +484,7 @@ iceberg.llap.only.query.files=\
   iceberg_merge_files.q,\
   iceberg_row_mode_virtual_columns.q,\
   llap_iceberg_read_orc.q,\
+  llap_iceberg_read_orc_deletes.q,\
   llap_iceberg_read_parquet.q,\
   puffin_col_stats_with_time_travel.q
 
diff --git 
a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/ColumnVectorBatch.java
 
b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/ColumnVectorBatch.java
index df9bcf2d7e1..fe1f041effe 100644
--- 
a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/ColumnVectorBatch.java
+++ 
b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/ColumnVectorBatch.java
@@ -34,6 +34,8 @@ public class ColumnVectorBatch {
   public MutableFilterContext filterContext;
   public ColumnVector[] cols;
   public int size;
+  /** File row position of the first row; Long.MIN_VALUE when the producer 
does not track positions. */
+  private long startRowInFile = Long.MIN_VALUE;
 
   public ColumnVectorBatch(int columnCount) {
     this(columnCount, VectorizedRowBatch.DEFAULT_SIZE);
@@ -51,6 +53,13 @@ public void swapColumnVector(int ix, ColumnVector[] other, 
int otherIx) {
     cols[ix] = old;
   }
 
+  public long getStartRowInFile() {
+    return startRowInFile;
+  }
+
+  public void setStartRowInFile(long startRowInFile) {
+    this.startRowInFile = startRowInFile;
+  }
   
   @Override
   public String toString() {
@@ -76,7 +85,6 @@ public String toString() {
     }
     b.append('\n');
 
-
     for (int i = 0; i < size; i++) {
       b.append('[');
       for (int k = 0; k < cols.length; k++) {
@@ -85,12 +93,10 @@ public String toString() {
           b.append(", ");
         }
         if (cv == null) continue;
-        if (cv != null) {
-          try {
-            cv.stringifyValue(b, i);
-          } catch (Exception ex) {
-            b.append("invalid");
-          }
+        try {
+          cv.stringifyValue(b, i);
+        } catch (Exception ex) {
+          b.append("invalid");
         }
       }
       b.append(']');
@@ -101,4 +107,4 @@ public String toString() {
 
     return b.toString();
   }
-}
\ No newline at end of file
+}
diff --git 
a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java
 
b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java
index d74efcd2d3b..7a2b26b00e6 100644
--- 
a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java
+++ 
b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java
@@ -53,6 +53,7 @@
 import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx;
 import org.apache.hadoop.hive.ql.io.AcidUtils;
 import org.apache.hadoop.hive.ql.io.BucketIdentifier;
+import org.apache.hadoop.hive.ql.io.RowPositionAwareVectorizedRecordReader;
 import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
 import org.apache.hadoop.hive.ql.io.orc.OrcRecordUpdater;
 import org.apache.hadoop.hive.ql.io.orc.OrcSplit;
@@ -85,7 +86,8 @@
 
 import static org.apache.hadoop.hive.llap.LlapHiveUtils.throwIfCacheOnlyRead;
 
-class LlapRecordReader implements RecordReader<NullWritable, 
VectorizedRowBatch>, Consumer<ColumnVectorBatch> {
+final class LlapRecordReader implements RecordReader<NullWritable, 
VectorizedRowBatch>, Consumer<ColumnVectorBatch>,
+    RowPositionAwareVectorizedRecordReader {
 
   private static final Logger LOG = 
LoggerFactory.getLogger(LlapRecordReader.class);
   private static final Object DONE_OBJECT = new Object();
@@ -96,7 +98,6 @@ class LlapRecordReader implements RecordReader<NullWritable, 
VectorizedRowBatch>
   private final SearchArgument sarg;
   private final VectorizedRowBatchCtx rbCtx;
   private final boolean isVectorized;
-  private final boolean probeDecodeEnabled;
   private VectorizedOrcAcidRowBatchReader acidReader;
   private Object[] partitionValues;
   private VectorizedRowBatch acidInputVrb;
@@ -222,8 +223,8 @@ private LlapRecordReader(MapWork mapWork, JobConf job, 
FileSplit split,
     this.includes = new IncludesImpl(tableIncludedCols, isAcidFormat, rbCtx,
         schema, job, isAcidScan && acidReader.includeAcidColumns());
 
-    this.probeDecodeEnabled = HiveConf.getBoolVar(jobConf, 
ConfVars.HIVE_OPTIMIZE_SCAN_PROBEDECODE);
-    if (this.probeDecodeEnabled) {
+    boolean probeDecodeEnabled = HiveConf.getBoolVar(jobConf, 
ConfVars.HIVE_OPTIMIZE_SCAN_PROBEDECODE);
+    if (probeDecodeEnabled) {
       includes.setProbeDecodeContext(mapWork.getProbeDecodeContext());
       LOG.info("LlapRecordReader ProbeDecode is enabled");
     }
@@ -301,34 +302,32 @@ static int determineQueueLimit(long maxBufferedSize,
       } else {
         PrimitiveTypeInfo pti = (PrimitiveTypeInfo) ti;
         switch (pti.getPrimitiveCategory()) {
-        case BINARY:
-        case CHAR:
-        case VARCHAR:
-        case STRING:
-          colWeight = COL_WEIGHT_STRING;
-          break;
+          case BINARY:
+          case CHAR:
+          case VARCHAR:
+          case STRING:
+            colWeight = COL_WEIGHT_STRING;
+            break;
           //Timestamp column vector uses an int and long arrays
-        case TIMESTAMP:
-        case INTERVAL_DAY_TIME:
-          colWeight = 2;
-          break;
-        case DECIMAL:
-          boolean useDecimal64 = false;
-          if (ti instanceof DecimalTypeInfo) {
-            DecimalTypeInfo dti = (DecimalTypeInfo) ti;
-            if (dti.getPrecision() <= TypeDescription.MAX_DECIMAL64_PRECISION 
&& decimal64Support) {
+          case TIMESTAMP:
+          case INTERVAL_DAY_TIME:
+            colWeight = 2;
+            break;
+          case DECIMAL:
+            boolean useDecimal64 = false;
+            if (ti instanceof DecimalTypeInfo dti &&
+                dti.getPrecision() <= TypeDescription.MAX_DECIMAL64_PRECISION 
&& decimal64Support) {
               useDecimal64 = true;
             }
-          }
-          // decimal_64 column vectors gets the same weight as long column 
vectors
-          if (useDecimal64) {
+            // decimal_64 column vectors gets the same weight as long column 
vectors
+            if (useDecimal64) {
+              colWeight = 1;
+            } else {
+              colWeight = COL_WEIGHT_HIVEDECIMAL;
+            }
+            break;
+          default:
             colWeight = 1;
-          } else {
-            colWeight = COL_WEIGHT_HIVEDECIMAL;
-          }
-          break;
-        default:
-          colWeight = 1;
         }
       }
       totalWeight += colWeight * 8 * scale;
@@ -500,7 +499,7 @@ private AcidWrapper(VectorizedRowBatch acidVrb) {
     }
 
     @Override
-    public boolean next(NullWritable key, VectorizedRowBatch value) throws 
IOException {
+    public boolean next(NullWritable key, VectorizedRowBatch value) {
       return true;
     }
 
@@ -515,16 +514,16 @@ public VectorizedRowBatch createValue() {
     }
 
     @Override
-    public long getPos() throws IOException {
+    public long getPos() {
       return 0;
     }
 
     @Override
-    public void close() throws IOException {
+    public void close() {
     }
 
     @Override
-    public float getProgress() throws IOException {
+    public float getProgress() {
       return 0;
     }
   }
@@ -581,6 +580,12 @@ ColumnVectorBatch nextCvb() throws InterruptedException, 
IOException {
     return lastCvb;
   }
 
+  /** Long.MIN_VALUE (unknown) when the producer does not track row positions. 
*/
+  @Override
+  public long getRowNumber() {
+    return lastCvb.getStartRowInFile();
+  }
+
   @Override
   public NullWritable createKey() {
     return NullWritable.get();
@@ -652,7 +657,11 @@ public void setError(Throwable t) throws 
InterruptedException {
 
   private void enqueueInternal(Object o) throws InterruptedException {
     // We need to loop here to handle the case where consumer goes away.
-    do {} while (!isClosed && !isInterrupted && !queue.offer(o, 100, 
TimeUnit.MILLISECONDS));
+    while (!isClosed && !isInterrupted) {
+      if (queue.offer(o, 100, TimeUnit.MILLISECONDS)) {
+        return;
+      }
+    }
   }
 
   @Override
@@ -665,10 +674,6 @@ void setPartitionValues(Object[] partitionValues) {
     this.partitionValues = partitionValues;
   }
 
-  ReadPipeline getReadPipeline() {
-    return rp;
-  }
-
   /** This class encapsulates include-related logic for LLAP readers. It is 
not actually specific
    *  to LLAP IO but in LLAP IO in particular, I want to encapsulate all this 
mess for now until
    *  we have smth better like Schema Evolution v2. This can also 
hypothetically encapsulate
diff --git 
a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/GenericColumnVectorProducer.java
 
b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/GenericColumnVectorProducer.java
index 49426eb7363..562cd973f14 100644
--- 
a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/GenericColumnVectorProducer.java
+++ 
b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/GenericColumnVectorProducer.java
@@ -21,6 +21,7 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.TimeZone;
@@ -59,6 +60,7 @@
 import org.apache.orc.OrcProto.RowIndex;
 import org.apache.orc.OrcProto.RowIndexEntry;
 import org.apache.orc.OrcProto.Type;
+import org.apache.orc.StripeInformation;
 import org.apache.orc.TypeDescription;
 
 public class GenericColumnVectorProducer implements ColumnVectorProducer {
@@ -304,5 +306,15 @@ public OrcFile.Version getFileVersion() {
     public CalendarKind getCalendar() {
       return CalendarKind.JULIAN_GREGORIAN;
     }
+
+    @Override
+    public List<StripeInformation> getStripes() {
+      return Collections.emptyList();
+    }
+
+    @Override
+    public int getRowIndexStride() {
+      return 0;
+    }
   }
 }
diff --git 
a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java
 
b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java
index 36e1ab3c54a..78e954fb1e4 100644
--- 
a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java
+++ 
b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java
@@ -70,6 +70,7 @@ public class OrcEncodedDataConsumer
   private TypeReader[] columnReaders;
   private int previousStripeIndex = -1;
   private ConsumerFileMetadata fileMetadata; // We assume one request is only 
for one file.
+  private long[] stripeStartRows; // row number in the file where each stripe 
starts; null without file positions
   private CompressionCodec codec;
   private List<ConsumerStripeMetadata> stripes;
   private SchemaEvolution evolution;
@@ -95,6 +96,12 @@ public void setUseDecimal64ColumnVectors(final boolean 
useDecimal64ColumnVectors
   public void setFileMetadata(ConsumerFileMetadata f) {
     assert fileMetadata == null;
     fileMetadata = f;
+    if (!f.getStripes().isEmpty()) {
+      stripeStartRows = new long[f.getStripes().size()];
+      for (int i = 1; i < stripeStartRows.length; ++i) {
+        stripeStartRows[i] = stripeStartRows[i - 1] + f.getStripes().get(i - 
1).getNumberOfRows();
+      }
+    }
     stripes = new ArrayList<>(f.getStripeCount());
     codec = WriterImpl.createCodec(fileMetadata.getCompressionKind());
   }
@@ -137,6 +144,13 @@ protected void decodeBatch(OrcEncodedColumnBatch batch,
       }
       int maxBatchesRG = (int) ((nonNullRowCount / 
VectorizedRowBatch.DEFAULT_SIZE) + 1);
       int batchSize = VectorizedRowBatch.DEFAULT_SIZE;
+      long rgStartRow = Long.MIN_VALUE;
+      if (stripeStartRows != null) {
+        rgStartRow = stripeStartRows[currentStripeIndex];
+        if (rgIdx != OrcEncodedColumnBatch.ALL_RGS) {
+          rgStartRow += (long) rgIdx * fileMetadata.getRowIndexStride();
+        }
+      }
       TypeDescription fileSchema = fileMetadata.getSchema();
 
       if (columnReaders == null || !sameStripe || noIndex) {
@@ -155,6 +169,8 @@ protected void decodeBatch(OrcEncodedColumnBatch batch,
 
         ColumnVectorBatch cvb = takeBatch(batchSize);
         // assert cvb.cols.length == batch.getColumnIxs().length; // Must be 
constant per split.
+        cvb.setStartRowInFile(stripeStartRows == null ?
+            Long.MIN_VALUE : rgStartRow + (long) i * 
VectorizedRowBatch.DEFAULT_SIZE);
         for (int idx = 0; idx < columnReaders.length; ++idx) {
           /*
            * Currently, ORC's TreeReaderFactory class does this:
@@ -302,11 +318,15 @@ private ColumnVector createColumn(TypeDescription type, 
int batchSize, final boo
   private void positionInStreams(TypeReader[] columnReaders,
       OrcBatchKey batchKey, ConsumerStripeMetadata stripeMetadata) throws 
IOException {
     PositionProvider[] pps = createPositionProviders(columnReaders, batchKey, 
stripeMetadata);
-    if (pps == null) return;
-    for (int i = 0; i < columnReaders.length; i++) {
-      if (columnReaders[i] == null) continue;
+    if (pps == null) {
+      return;
+    }
+    for (TypeReader columnReader : columnReaders) {
+      if (columnReader == null) {
+        continue;
+      }
       // TODO: we could/should trace seek destinations; pps needs a "peek" 
method
-      columnReaders[i].seek(pps, TypeReader.ReadPhase.ALL);
+      columnReader.seek(pps, TypeReader.ReadPhase.ALL);
     }
   }
 
@@ -318,9 +338,10 @@ private void repositionInStreams(TypeReader[] 
columnReaders,
     if (LlapIoImpl.ORC_LOGGER.isTraceEnabled()) {
       LlapIoImpl.ORC_LOGGER.trace("Created pps {}", Arrays.toString(pps));
     }
-    if (pps == null) return;
-    for (int i = 0; i < columnReaders.length; i++) {
-      TypeReader reader = columnReaders[i];
+    if (pps == null) {
+      return;
+    }
+    for (TypeReader reader : columnReaders) {
       if (reader == null) continue;
       // Note: we assume this never happens for SerDe reader - the batch would 
never have vectors.
       // That is always true now; but it wasn't some day, the below would 
throw in getColumnData.
@@ -329,7 +350,7 @@ private void repositionInStreams(TypeReader[] columnReaders,
       // SettableTreeReader so that we can avoid this check.
       if (reader instanceof EncodedTreeReaderFactory.TimestampStreamReader && 
!sameStripe) {
         ((EncodedTreeReaderFactory.TimestampStreamReader) reader)
-                .updateTimezone(stripeMetadata.getWriterTimezone());
+            .updateTimezone(stripeMetadata.getWriterTimezone());
       }
       reader.seek(pps, TypeReader.ReadPhase.ALL);
     }
@@ -359,9 +380,7 @@ private PositionProvider[] createPositionProviders(
     if (!stripeMetadata.supportsRowIndexes()) {
       PositionProvider singleRgPp = new IndexlessPositionProvider();
       pps = new PositionProvider[stripeMetadata.getEncodings().size()];
-      for (int i = 0; i < pps.length; ++i) {
-        pps[i] = singleRgPp;
-      }
+      Arrays.fill(pps, singleRgPp);
     } else {
       int rowGroupIndex = batchKey.rgIx;
       if (rowGroupIndex == OrcEncodedColumnBatch.ALL_RGS) {
diff --git 
a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/ConsumerFileMetadata.java
 
b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/ConsumerFileMetadata.java
index 928b4833675..59a3ccf13e3 100644
--- 
a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/ConsumerFileMetadata.java
+++ 
b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/ConsumerFileMetadata.java
@@ -25,6 +25,7 @@
 import org.apache.orc.OrcFile;
 import org.apache.orc.OrcProto.CalendarKind;
 import org.apache.orc.OrcProto.Type;
+import org.apache.orc.StripeInformation;
 import org.apache.orc.TypeDescription;
 
 public interface ConsumerFileMetadata {
@@ -34,4 +35,7 @@ public interface ConsumerFileMetadata {
   TypeDescription getSchema() throws FileFormatException;
   OrcFile.Version getFileVersion();
   CalendarKind getCalendar();
+  /** Every stripe of the file in order, or empty when the data has no file 
positions (re-encoded input). */
+  List<StripeInformation> getStripes();
+  int getRowIndexStride();
 }
diff --git 
a/llap-server/src/test/org/apache/hadoop/hive/llap/io/decode/TestOrcEncodedDataConsumerRowPositions.java
 
b/llap-server/src/test/org/apache/hadoop/hive/llap/io/decode/TestOrcEncodedDataConsumerRowPositions.java
new file mode 100644
index 00000000000..45c8ad6d33f
--- /dev/null
+++ 
b/llap-server/src/test/org/apache/hadoop/hive/llap/io/decode/TestOrcEncodedDataConsumerRowPositions.java
@@ -0,0 +1,299 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hive.llap.io.decode;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static 
org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_NAME;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.llap.counters.QueryFragmentCounters;
+import org.apache.hadoop.hive.llap.io.api.LlapProxy;
+import org.apache.hadoop.hive.llap.io.api.impl.ColumnVectorBatch;
+import org.apache.hadoop.hive.llap.io.decode.ColumnVectorProducer.Includes;
+import 
org.apache.hadoop.hive.llap.io.decode.GenericColumnVectorProducer.SerDeStripeMetadata;
+import org.apache.hadoop.hive.llap.io.metadata.ConsumerFileMetadata;
+import org.apache.hadoop.hive.llap.metrics.LlapDaemonIOMetrics;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx;
+import org.apache.hadoop.hive.ql.io.IOConstants;
+import org.apache.hadoop.hive.ql.io.RowPositionAwareVectorizedRecordReader;
+import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
+import org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat;
+import org.apache.hadoop.hive.ql.io.orc.encoded.Consumer;
+import org.apache.hadoop.hive.ql.io.orc.encoded.IoTrace;
+import org.apache.hadoop.hive.ql.io.orc.encoded.Reader.OrcEncodedColumnBatch;
+import org.apache.hadoop.hive.ql.plan.MapWork;
+import org.apache.hadoop.hive.ql.plan.PartitionDesc;
+import org.apache.hadoop.hive.ql.plan.TableDesc;
+import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RecordReader;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.orc.CompressionKind;
+import org.apache.orc.OrcFile;
+import org.apache.orc.OrcProto.ColumnEncoding;
+import org.apache.orc.StripeInformation;
+import org.apache.orc.TypeDescription;
+import org.apache.orc.Writer;
+import org.apache.tez.common.counters.TezCounters;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Absolute file row positions reported by {@link OrcEncodedDataConsumer} on 
every emitted batch.
+ * ORC files are read end to end through LLAP IO; the position-less SerDe case 
is driven straight
+ * into decodeBatch.
+ */
+public class TestOrcEncodedDataConsumerRowPositions {
+
+  private static final TypeDescription SCHEMA = 
TypeDescription.fromString("struct<id:bigint,name:string>");
+  private static final int STRIDE = 2500;
+  // Neither a multiple of the row index stride nor of 
VectorizedRowBatch.DEFAULT_SIZE.
+  private static final int[] STRIPE_ROWS = {8300, 6100, 3700};
+  private static final int[] NO_INDEX_STRIPE_ROWS = {5300, 2700};
+  private static final List<Integer> ALL_COLUMNS = Arrays.asList(0, 1);
+
+  private static final LlapDaemonIOMetrics IO_METRICS =
+      LlapDaemonIOMetrics.create("TestOrcEncodedDataConsumerRowPositions-io", 
"1", null);
+
+  private static HiveConf daemonConf;
+  private static Path indexedFile;
+  private static Path noIndexFile;
+
+  @BeforeClass
+  public static void setUpClass() throws Exception {
+    daemonConf = new HiveConf();
+    HiveConf.setVar(daemonConf, ConfVars.LLAP_IO_MEMORY_MODE, "none");
+
+    Path tmpDir = new 
Path(Files.createTempDirectory("llap-orc-row-positions").toString());
+    indexedFile = new Path(tmpDir, "indexed.orc");
+    noIndexFile = new Path(tmpDir, "no-index.orc");
+    writeFile(indexedFile, STRIDE, STRIPE_ROWS);
+    writeFile(noIndexFile, 0, NO_INDEX_STRIPE_ROWS);
+    assertLayout(indexedFile, STRIDE, STRIPE_ROWS);
+    assertLayout(noIndexFile, 0, NO_INDEX_STRIPE_ROWS);
+
+    LlapProxy.setDaemon(true);
+    LlapProxy.initializeLlapIo(daemonConf);
+  }
+
+  @AfterClass
+  public static void tearDownClass() {
+    LlapProxy.close();
+  }
+
+  @Test
+  public void testRowGroupPositionsAcrossStripes() throws Exception {
+    assertEquals(expectedBatches(STRIDE, STRIPE_ROWS), 
readThroughLlap(indexedFile, ALL_COLUMNS));
+  }
+
+  @Test
+  public void testIndexDisabledFileStartsAtStripeBoundary() throws Exception {
+    assertEquals(expectedBatches(Integer.MAX_VALUE, NO_INDEX_STRIPE_ROWS), 
readThroughLlap(noIndexFile, ALL_COLUMNS));
+  }
+
+  /** Without columns, the reader decodes each stripe as one ALL_RGS batch. */
+  @Test
+  public void testNoColumnReadStartsAtStripeBoundary() throws Exception {
+    assertEquals(expectedBatches(Integer.MAX_VALUE, STRIPE_ROWS),
+        readThroughLlap(indexedFile, Collections.emptyList()));
+  }
+
+  @Test
+  public void testNoStripesReportsNoPosition() throws Exception {
+    assertEquals(Collections.nCopies(3, Long.MIN_VALUE), decodeStripe(2 * 
VectorizedRowBatch.DEFAULT_SIZE + 1));
+  }
+
+  // ---- LLAP IO end to end ----
+
+  /**
+   * "startRowInFile:size" per batch, in the order LLAP returned them. When 
columns are read, the first row of each
+   * batch must be the row at the reported position.
+   */
+  private static List<String> readThroughLlap(Path path, List<Integer> 
columnIds) throws Exception {
+    RecordReader<NullWritable, VectorizedRowBatch> reader = llapReader(path, 
columnIds);
+    List<String> batches = new ArrayList<>();
+    try {
+      VectorizedRowBatch vrb = reader.createValue();
+      while (reader.next(NullWritable.get(), vrb)) {
+        long position = ((RowPositionAwareVectorizedRecordReader) 
reader).getRowNumber();
+        if (!columnIds.isEmpty()) {
+          assertEquals("id of the first row of the batch at " + position,
+              position, ((LongColumnVector) vrb.cols[0]).vector[0]);
+        }
+        batches.add(position + ":" + vrb.size);
+      }
+    } finally {
+      reader.close();
+    }
+    return batches;
+  }
+
+  private static RecordReader<NullWritable, VectorizedRowBatch> 
llapReader(Path path, List<Integer> columnIds)
+      throws IOException {
+    JobConf job = jobConf(path);
+    long fileLen = path.getFileSystem(job).getFileStatus(path).getLen();
+    RecordReader<NullWritable, VectorizedRowBatch> reader = 
LlapProxy.getIo().orc().vectorizedReaderForPath(
+        null, path, null, columnIds, job, 0, fileLen, Reporter.NULL);
+    assertNotNull("LLAP should handle this ORC read", reader);
+    return reader;
+  }
+
+  // ---- decodeBatch without file positions ----
+
+  /** Row positions of the batches decodeBatch emits for a zero-column read of 
one SerDe stripe. */
+  private static List<Long> decodeStripe(int stripeRows) throws Exception {
+    // No stripes, as for SerDe-encoded input.
+    ConsumerFileMetadata fileMetadata = mock(ConsumerFileMetadata.class);
+    when(fileMetadata.getCompressionKind()).thenReturn(CompressionKind.NONE);
+    SerDeStripeMetadata stripeMetadata = new SerDeStripeMetadata(0);
+    stripeMetadata.setRowCount(stripeRows);
+    stripeMetadata.setEncodings(Collections.singletonList(
+        
ColumnEncoding.newBuilder().setKind(ColumnEncoding.Kind.DIRECT).build()));
+    Includes includes = mock(Includes.class);
+    when(includes.getBatchReaderTypes(any())).thenReturn(new 
TypeDescription[0]);
+
+    QueryFragmentCounters counters = new QueryFragmentCounters(daemonConf, new 
TezCounters());
+    OrcEncodedDataConsumer consumer = new OrcEncodedDataConsumer(null, 
includes, counters, IO_METRICS);
+    consumer.init(null, null, new IoTrace(0, false));
+    consumer.setFileMetadata(fileMetadata);
+    consumer.setStripeMetadata(stripeMetadata);
+
+    List<Long> starts = new ArrayList<>();
+    OrcEncodedColumnBatch batch = new OrcEncodedColumnBatch();
+    batch.init(null, 0, OrcEncodedColumnBatch.ALL_RGS, 0);
+    consumer.decodeBatch(batch, new Consumer<>() {
+      @Override
+      public void consumeData(ColumnVectorBatch cvb) {
+        starts.add(cvb.getStartRowInFile());
+      }
+
+      @Override
+      public void setDone() {
+        // decodeBatch does not call it
+      }
+
+      @Override
+      public void setError(Throwable t) {
+        throw new AssertionError(t);
+      }
+    });
+    return starts;
+  }
+
+  // ---- fixture ----
+
+  private static void writeFile(Path path, int stride, int[] stripeRows) 
throws IOException {
+    try (Writer writer = OrcFile.createWriter(path, 
OrcFile.writerOptions(daemonConf)
+        
.setSchema(SCHEMA).rowIndexStride(stride).compress(CompressionKind.NONE))) {
+      VectorizedRowBatch batch = 
SCHEMA.createRowBatch(Arrays.stream(stripeRows).max().getAsInt());
+      LongColumnVector ids = (LongColumnVector) batch.cols[0];
+      BytesColumnVector names = (BytesColumnVector) batch.cols[1];
+      long row = 0;
+      for (int s = 0; s < stripeRows.length; ++s) {
+        batch.reset();
+        batch.size = stripeRows[s];
+        for (int i = 0; i < batch.size; ++i, ++row) {
+          ids.vector[i] = row;
+          names.setVal(i, ("n" + row).getBytes(UTF_8));
+        }
+        writer.addRowBatch(batch);
+        if (s < stripeRows.length - 1) {
+          writer.writeIntermediateFooter();
+        }
+      }
+    }
+  }
+
+  private static void assertLayout(Path path, int stride, int[] stripeRows) 
throws IOException {
+    try (org.apache.orc.Reader reader = OrcFile.createReader(path, 
OrcFile.readerOptions(daemonConf))) {
+      assertEquals(stride, reader.getRowIndexStride());
+      List<StripeInformation> stripes = reader.getStripes();
+      assertEquals(stripeRows.length, stripes.size());
+      for (int i = 0; i < stripeRows.length; ++i) {
+        assertEquals("stripe " + i, stripeRows[i], 
stripes.get(i).getNumberOfRows());
+      }
+    }
+  }
+
+  private static JobConf jobConf(Path path) {
+    JobConf job = new JobConf(daemonConf);
+    HiveConf.setBoolVar(job, ConfVars.HIVE_VECTORIZATION_ENABLED, true);
+    HiveConf.setVar(job, ConfVars.PLAN, "//tmp");
+    job.set(IOConstants.COLUMNS, "id,name");
+    job.set(IOConstants.COLUMNS_TYPES, "bigint,string");
+    job.set(ColumnProjectionUtils.ORC_SCHEMA_STRING, SCHEMA.toString());
+
+    Properties tblProps = new Properties();
+    tblProps.setProperty(META_TABLE_NAME, "default.test_orc_positions");
+    TableDesc tableDesc = new TableDesc(OrcInputFormat.class, 
OrcOutputFormat.class, tblProps);
+
+    MapWork mapWork = new MapWork();
+    mapWork.setVectorMode(true);
+    mapWork.setVectorizedRowBatchCtx(new VectorizedRowBatchCtx(
+        new String[] {"id", "name"},
+        new TypeInfo[] {TypeInfoFactory.longTypeInfo, 
TypeInfoFactory.stringTypeInfo},
+        null, null, 0, 0, null, new String[0], null));
+    PartitionDesc partitionDesc = new PartitionDesc();
+    partitionDesc.setTableDesc(tableDesc);
+    mapWork.addPathToPartitionInfo(path.getParent(), partitionDesc);
+    Utilities.setMapWork(job, mapWork);
+    return job;
+  }
+
+  // ---- expectations ----
+
+  /** "startRowInFile:size" of every batch of a file whose stripes are read 
whole. */
+  private static List<String> expectedBatches(int stride, int[] stripeRows) {
+    List<String> expected = new ArrayList<>();
+    long base = 0;
+    for (int rows : stripeRows) {
+      for (int rgStart = 0; rgStart < rows; rgStart += stride) {
+        int rgRows = Math.min(stride, rows - rgStart);
+        for (int off = 0; off < rgRows; off += 
VectorizedRowBatch.DEFAULT_SIZE) {
+          expected.add((base + rgStart + off) + ":" + 
Math.min(VectorizedRowBatch.DEFAULT_SIZE, rgRows - off));
+        }
+      }
+      base += rows;
+    }
+    return expected;
+  }
+}

Reply via email to