This is an automated email from the ASF dual-hosted git repository.
deniskuzZ pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 875607b9ee4 HIVE-30102: Iceberg: LLAP cannot read an ORC file that has
positional deletes (#6920)
875607b9ee4 is described below
commit 875607b9ee452bbfc4b2fec81ea217c039041434
Author: Denys Kuzmenko <[email protected]>
AuthorDate: Sat Oct 31 12:49:27 2026 +0100
HIVE-30102: Iceberg: LLAP cannot read an ORC file that has positional
deletes (#6920)
---
.../mr/hive/vector/HiveVectorizedReader.java | 6 +-
.../iceberg/mr/mapreduce/IcebergRecordReader.java | 3 +
.../iceberg/mr/hive/TestHiveIcebergCRUD.java | 26 ++
.../hive/vector/TestHiveIcebergVectorization.java | 25 ++
.../positive/llap_iceberg_read_orc_deletes.q | 26 ++
.../llap/llap_iceberg_read_orc_deletes.q.out | 138 ++++++++++
.../test/resources/testconfiguration.properties | 2 +
.../hive/llap/io/api/impl/ColumnVectorBatch.java | 22 +-
.../hive/llap/io/api/impl/LlapRecordReader.java | 79 +++---
.../io/decode/GenericColumnVectorProducer.java | 12 +
.../llap/io/decode/OrcEncodedDataConsumer.java | 41 ++-
.../llap/io/metadata/ConsumerFileMetadata.java | 4 +
.../TestOrcEncodedDataConsumerRowPositions.java | 299 +++++++++++++++++++++
13 files changed, 624 insertions(+), 59 deletions(-)
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveVectorizedReader.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveVectorizedReader.java
index 52140dfb88b..99a609a36a4 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveVectorizedReader.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveVectorizedReader.java
@@ -61,7 +61,6 @@
import org.apache.iceberg.io.CloseableIterable;
import org.apache.iceberg.io.CloseableIterator;
import org.apache.iceberg.io.FileIO;
-import org.apache.iceberg.mr.InputFormatConfig;
import org.apache.iceberg.mr.hive.HiveIcebergInputFormat;
import org.apache.iceberg.mr.mapred.MapredIcebergInputFormat;
import org.apache.iceberg.orc.VectorizedReadUtils;
@@ -220,6 +219,8 @@ private static RecordReader<NullWritable,
VectorizedRowBatch> orcRecordReader(Jo
// Need to turn positional schema evolution off since we use column name
based schema evolution for projection
// and Iceberg will make a mapping between the file schema and the current
reading schema.
job.setBoolean(OrcConf.FORCE_POSITIONAL_EVOLUTION.getHiveConfName(),
false);
+ // Row positions are derived from the batch index, and ORC's row-level
filter can drop rows and whole batches.
+ job.setBoolean(OrcConf.ALLOW_SARG_TO_FILTER.getHiveConfName(), false);
// Metadata information has to be passed along in the OrcSplit. Without
specifying this, the vectorized
// reader will assume that the ORC file ends at the task's start + length,
and might fail reading the tail..
@@ -230,9 +231,8 @@ private static RecordReader<NullWritable,
VectorizedRowBatch> orcRecordReader(Jo
VectorizedReadUtils.deserializeToShadedOrcTail(serializedOrcTail).getSchema(),
residual);
// If LLAP enabled, try to retrieve an LLAP record reader - this might
yield to null in some special cases
- // TODO: add support for reading files with positional deletes with LLAP
(LLAP would need to provide file row num)
if (HiveConf.getBoolVar(job, HiveConf.ConfVars.LLAP_IO_ENABLED,
LlapProxy.isDaemon()) &&
- LlapProxy.getIo() != null && task.deletes().isEmpty() &&
!InputFormatConfig.fetchVirtualColumns(job)) {
+ LlapProxy.getIo() != null) {
recordReader = LlapProxy.getIo().orc().vectorizedReaderForPath(fileId,
path, null, readColumnIds,
job, start, length, reporter);
}
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java
index 2ea261f2014..52faaf20230 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java
@@ -34,6 +34,7 @@
import org.apache.hadoop.hive.ql.io.IOContextAwareRecordReader;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hive.iceberg.org.apache.orc.OrcConf;
import org.apache.iceberg.DataFile;
import org.apache.iceberg.DataTask;
import org.apache.iceberg.FileFormat;
@@ -262,6 +263,8 @@ private CloseableIterable<T> newOrcIterable(
ORC.ReadBuilder orcReadBuilder = ORC.read(inputFile)
.project(readSchemaWithoutConstantAndMetadataFields)
.filter(residual)
+ // Row positions are derived from the batch index, and ORC's row-level
filter can drop rows and whole batches.
+ .config(OrcConf.ALLOW_SARG_TO_FILTER.getHiveConfName(), "false")
.caseSensitive(isCaseSensitive())
.split(task.start(), task.length());
diff --git
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergCRUD.java
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergCRUD.java
index e3236fe2c3e..ee4d285a9ab 100644
---
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergCRUD.java
+++
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergCRUD.java
@@ -215,6 +215,32 @@ public void
testReadAndWriteFormatV2Partitioned_PosDelete() throws IOException {
Assert.assertArrayEquals(new Object[] {2L, "Trudy", "Pink"},
objects.get(3));
}
+ /**
+ * ORC can filter rows by the pushed down predicate; in row mode, row
positions and deletes must still apply to
+ * the right rows.
+ */
+ @Test
+ public void testPositionDeleteWithOrcRowFiltering() {
+ Assume.assumeTrue(fileFormat == FileFormat.ORC && formatVersion >= 2 &&
!isVectorized);
+
+ Schema schema = new Schema(optional(1, "customer_id",
Types.LongType.get()));
+ List<Record> records = TestHelper.generateRandomRecords(schema, 30000, 0L);
+ for (int i = 0; i < records.size(); ++i) {
+ records.get(i).setField("customer_id", (long) i);
+ }
+ testTables.createTable(shell, "customers", schema,
PartitionSpec.unpartitioned(), fileFormat, records,
+ formatVersion);
+ shell.executeStatement("DELETE FROM customers WHERE customer_id = 25000");
+
+ shell.setHiveSessionValue("orc.sarg.to.filter", true);
+ shell.setHiveSessionValue("orc.filter.use.selected", true);
+ List<Object[]> result = shell.executeStatement(
+ "SELECT customer_id, ROW__POSITION FROM customers WHERE customer_id IN
(25000, 25001)");
+
+ Assert.assertEquals(1, result.size());
+ Assert.assertArrayEquals(new Object[] {25001L, 25001L}, result.get(0));
+ }
+
/**
* Tests reading/consuming legacy Iceberg V2 tables containing position
delete files with embedded
* row data. Iceberg 1.12.0 removed writer support for row data in position
deletes, but Hive
diff --git
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/vector/TestHiveIcebergVectorization.java
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/vector/TestHiveIcebergVectorization.java
index 28487b715ff..e57ab182367 100644
---
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/vector/TestHiveIcebergVectorization.java
+++
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/vector/TestHiveIcebergVectorization.java
@@ -219,6 +219,31 @@ private void
testVectorizedReadWithDeleteFilter(Map<String, String> props) {
validation.apply(1501);
}
+ /**
+ * ORC can filter the rows of a batch by the pushed down predicate; row
positions and deletes must still apply to
+ * the right rows.
+ */
+ @Test
+ public void testHiveDeleteFilterWithOrcRowFiltering() {
+ Assume.assumeTrue(fileFormat == FileFormat.ORC);
+
+ Schema schema = new Schema(optional(1, "customer_id",
Types.LongType.get()));
+ List<Record> records = TestHelper.generateRandomRecords(schema, 30000, 0L);
+ for (int i = 0; i < records.size(); ++i) {
+ records.get(i).setField("customer_id", (long) i);
+ }
+ testTables.createTable(shell, "vectordelete", schema,
PartitionSpec.unpartitioned(), fileFormat, records, 2);
+ shell.executeStatement("DELETE FROM vectordelete WHERE customer_id =
25000");
+
+ shell.setHiveSessionValue("orc.sarg.to.filter", true);
+ shell.setHiveSessionValue("orc.filter.use.selected", true);
+ List<Object[]> result = shell.executeStatement(
+ "select customer_id, ROW__POSITION from vectordelete where customer_id
in (25000, 25001)");
+
+ Assert.assertEquals(1, result.size());
+ Assert.assertArrayEquals(new Object[] { 25001L, 25001L }, result.get(0));
+ }
+
@Test
public void testHiveDeleteFilterWithFilteredParquetBlock() {
Assume.assumeTrue(fileFormat == FileFormat.PARQUET);
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/llap_iceberg_read_orc_deletes.q
b/iceberg/iceberg-handler/src/test/queries/positive/llap_iceberg_read_orc_deletes.q
new file mode 100644
index 00000000000..bc37f0a23c4
--- /dev/null
+++
b/iceberg/iceberg-handler/src/test/queries/positive/llap_iceberg_read_orc_deletes.q
@@ -0,0 +1,26 @@
+-- ORC files with position deletes are read through LLAP IO (see the LLAP IO
counters): the deletes and ROW__POSITION
+-- apply to the right rows in a later row group and across a batch boundary
(the third row group starts at file row
+-- 20000, its sixth batch at 25120)
+set hive.llap.io.enabled=true;
+set hive.vectorized.execution.enabled=true;
+set hive.fetch.task.conversion=none;
+set hive.optimize.index.filter=true;
+
+-- V2: position delete files
+CREATE EXTERNAL TABLE ice_orc_v2 (id INT) STORED BY ICEBERG STORED AS ORC
+TBLPROPERTIES ('format-version'='2');
+INSERT INTO ice_orc_v2 SELECT pos FROM (SELECT posexplode(split(space(29999),
' ')) AS (pos, val)) t;
+DELETE FROM ice_orc_v2 WHERE id IN (25119, 25121);
+
+-- V3: deletion vectors
+CREATE EXTERNAL TABLE ice_orc_v3 (id INT) STORED BY ICEBERG STORED AS ORC
+TBLPROPERTIES ('format-version'='3');
+INSERT INTO ice_orc_v3 SELECT pos FROM (SELECT posexplode(split(space(29999),
' ')) AS (pos, val)) t;
+DELETE FROM ice_orc_v3 WHERE id IN (25119, 25121);
+
+SET
hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.PostExecTezSummaryPrinter;
+SELECT id, ROW__POSITION FROM ice_orc_v2 WHERE id BETWEEN 25117 AND 25123
ORDER BY id;
+SELECT id, ROW__POSITION FROM ice_orc_v3 WHERE id BETWEEN 25117 AND 25123
ORDER BY id;
+
+DROP TABLE ice_orc_v2;
+DROP TABLE ice_orc_v3;
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/llap/llap_iceberg_read_orc_deletes.q.out
b/iceberg/iceberg-handler/src/test/results/positive/llap/llap_iceberg_read_orc_deletes.q.out
new file mode 100644
index 00000000000..d49ae17cb09
--- /dev/null
+++
b/iceberg/iceberg-handler/src/test/results/positive/llap/llap_iceberg_read_orc_deletes.q.out
@@ -0,0 +1,138 @@
+PREHOOK: query: CREATE EXTERNAL TABLE ice_orc_v2 (id INT) STORED BY ICEBERG
STORED AS ORC
+TBLPROPERTIES ('format-version'='2')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ice_orc_v2
+POSTHOOK: query: CREATE EXTERNAL TABLE ice_orc_v2 (id INT) STORED BY ICEBERG
STORED AS ORC
+TBLPROPERTIES ('format-version'='2')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ice_orc_v2
+PREHOOK: query: INSERT INTO ice_orc_v2 SELECT pos FROM (SELECT
posexplode(split(space(29999), ' ')) AS (pos, val)) t
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc_v2
+POSTHOOK: query: INSERT INTO ice_orc_v2 SELECT pos FROM (SELECT
posexplode(split(space(29999), ' ')) AS (pos, val)) t
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc_v2
+PREHOOK: query: DELETE FROM ice_orc_v2 WHERE id IN (25119, 25121)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_orc_v2
+PREHOOK: Output: default@ice_orc_v2
+POSTHOOK: query: DELETE FROM ice_orc_v2 WHERE id IN (25119, 25121)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_orc_v2
+POSTHOOK: Output: default@ice_orc_v2
+PREHOOK: query: CREATE EXTERNAL TABLE ice_orc_v3 (id INT) STORED BY ICEBERG
STORED AS ORC
+TBLPROPERTIES ('format-version'='3')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ice_orc_v3
+POSTHOOK: query: CREATE EXTERNAL TABLE ice_orc_v3 (id INT) STORED BY ICEBERG
STORED AS ORC
+TBLPROPERTIES ('format-version'='3')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ice_orc_v3
+PREHOOK: query: INSERT INTO ice_orc_v3 SELECT pos FROM (SELECT
posexplode(split(space(29999), ' ')) AS (pos, val)) t
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc_v3
+POSTHOOK: query: INSERT INTO ice_orc_v3 SELECT pos FROM (SELECT
posexplode(split(space(29999), ' ')) AS (pos, val)) t
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc_v3
+PREHOOK: query: DELETE FROM ice_orc_v3 WHERE id IN (25119, 25121)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_orc_v3
+PREHOOK: Output: default@ice_orc_v3
+POSTHOOK: query: DELETE FROM ice_orc_v3 WHERE id IN (25119, 25121)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_orc_v3
+POSTHOOK: Output: default@ice_orc_v3
+PREHOOK: query: SELECT id, ROW__POSITION FROM ice_orc_v2 WHERE id BETWEEN
25117 AND 25123 ORDER BY id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_orc_v2
+#### A masked pattern was here ####
+Stage-1 FILE SYSTEM COUNTERS:
+Stage-1 HIVE COUNTERS:
+ CREATED_FILES: 1
+ DESERIALIZE_ERRORS: 0
+ RECORDS_IN_Map_1: 9998
+ RECORDS_OUT_0: 5
+ RECORDS_OUT_INTERMEDIATE_Map_1: 5
+ RECORDS_OUT_INTERMEDIATE_Reducer_2: 0
+ RECORDS_OUT_OPERATOR_FIL_7: 5
+ RECORDS_OUT_OPERATOR_FS_11: 5
+ RECORDS_OUT_OPERATOR_MAP_0: 0
+ RECORDS_OUT_OPERATOR_RS_9: 5
+ RECORDS_OUT_OPERATOR_SEL_10: 5
+ RECORDS_OUT_OPERATOR_SEL_8: 5
+ RECORDS_OUT_OPERATOR_TS_0: 9998
+ TOTAL_TABLE_ROWS_WRITTEN: 0
+Stage-1 LLAP IO COUNTERS:
+ CACHE_HIT_BYTES: 229
+ CACHE_MISS_BYTES: 0
+ METADATA_CACHE_HIT: 2
+ NUM_DECODED_BATCHES: 1
+ NUM_VECTOR_BATCHES: 10
+ ROWS_EMITTED: 10000
+ SELECTED_ROWGROUPS: 1
+Stage-1 INPUT COUNTERS:
+ GROUPED_INPUT_SPLITS_Map_1: 1
+ INPUT_DIRECTORIES_Map_1: 1
+ INPUT_FILES_Map_1: 1
+ RAW_INPUT_SPLITS_Map_1: 1
+25117 25117
+25118 25118
+25120 25120
+25122 25122
+25123 25123
+PREHOOK: query: SELECT id, ROW__POSITION FROM ice_orc_v3 WHERE id BETWEEN
25117 AND 25123 ORDER BY id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_orc_v3
+#### A masked pattern was here ####
+Stage-1 FILE SYSTEM COUNTERS:
+Stage-1 HIVE COUNTERS:
+ CREATED_FILES: 1
+ DESERIALIZE_ERRORS: 0
+ RECORDS_IN_Map_1: 9998
+ RECORDS_OUT_0: 5
+ RECORDS_OUT_INTERMEDIATE_Map_1: 5
+ RECORDS_OUT_INTERMEDIATE_Reducer_2: 0
+ RECORDS_OUT_OPERATOR_FIL_7: 5
+ RECORDS_OUT_OPERATOR_FS_11: 5
+ RECORDS_OUT_OPERATOR_MAP_0: 0
+ RECORDS_OUT_OPERATOR_RS_9: 5
+ RECORDS_OUT_OPERATOR_SEL_10: 5
+ RECORDS_OUT_OPERATOR_SEL_8: 5
+ RECORDS_OUT_OPERATOR_TS_0: 9998
+ TOTAL_TABLE_ROWS_WRITTEN: 0
+Stage-1 LLAP IO COUNTERS:
+ CACHE_HIT_BYTES: 229
+ CACHE_MISS_BYTES: 0
+ METADATA_CACHE_HIT: 2
+ NUM_DECODED_BATCHES: 1
+ NUM_VECTOR_BATCHES: 10
+ ROWS_EMITTED: 10000
+ SELECTED_ROWGROUPS: 1
+Stage-1 INPUT COUNTERS:
+ GROUPED_INPUT_SPLITS_Map_1: 1
+ INPUT_DIRECTORIES_Map_1: 1
+ INPUT_FILES_Map_1: 1
+ RAW_INPUT_SPLITS_Map_1: 1
+25117 25117
+25118 25118
+25120 25120
+25122 25122
+25123 25123
+PREHOOK: query: DROP TABLE ice_orc_v2
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@ice_orc_v2
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ice_orc_v2
+PREHOOK: query: DROP TABLE ice_orc_v3
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@ice_orc_v3
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ice_orc_v3
diff --git a/itests/src/test/resources/testconfiguration.properties
b/itests/src/test/resources/testconfiguration.properties
index e375f4f16e7..8de15596c5d 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -423,6 +423,7 @@ iceberg.llap.query.files=\
iceberg_merge_files.q,\
iceberg_row_mode_virtual_columns.q,\
llap_iceberg_read_orc.q,\
+ llap_iceberg_read_orc_deletes.q,\
llap_iceberg_read_parquet.q,\
puffin_col_stats_with_time_travel.q,\
vectorized_iceberg_read_mixed.q,\
@@ -483,6 +484,7 @@ iceberg.llap.only.query.files=\
iceberg_merge_files.q,\
iceberg_row_mode_virtual_columns.q,\
llap_iceberg_read_orc.q,\
+ llap_iceberg_read_orc_deletes.q,\
llap_iceberg_read_parquet.q,\
puffin_col_stats_with_time_travel.q
diff --git
a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/ColumnVectorBatch.java
b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/ColumnVectorBatch.java
index df9bcf2d7e1..fe1f041effe 100644
---
a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/ColumnVectorBatch.java
+++
b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/ColumnVectorBatch.java
@@ -34,6 +34,8 @@ public class ColumnVectorBatch {
public MutableFilterContext filterContext;
public ColumnVector[] cols;
public int size;
+ /** File row position of the first row; Long.MIN_VALUE when the producer
does not track positions. */
+ private long startRowInFile = Long.MIN_VALUE;
public ColumnVectorBatch(int columnCount) {
this(columnCount, VectorizedRowBatch.DEFAULT_SIZE);
@@ -51,6 +53,13 @@ public void swapColumnVector(int ix, ColumnVector[] other,
int otherIx) {
cols[ix] = old;
}
+ public long getStartRowInFile() {
+ return startRowInFile;
+ }
+
+ public void setStartRowInFile(long startRowInFile) {
+ this.startRowInFile = startRowInFile;
+ }
@Override
public String toString() {
@@ -76,7 +85,6 @@ public String toString() {
}
b.append('\n');
-
for (int i = 0; i < size; i++) {
b.append('[');
for (int k = 0; k < cols.length; k++) {
@@ -85,12 +93,10 @@ public String toString() {
b.append(", ");
}
if (cv == null) continue;
- if (cv != null) {
- try {
- cv.stringifyValue(b, i);
- } catch (Exception ex) {
- b.append("invalid");
- }
+ try {
+ cv.stringifyValue(b, i);
+ } catch (Exception ex) {
+ b.append("invalid");
}
}
b.append(']');
@@ -101,4 +107,4 @@ public String toString() {
return b.toString();
}
-}
\ No newline at end of file
+}
diff --git
a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java
b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java
index d74efcd2d3b..7a2b26b00e6 100644
---
a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java
+++
b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java
@@ -53,6 +53,7 @@
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx;
import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.io.BucketIdentifier;
+import org.apache.hadoop.hive.ql.io.RowPositionAwareVectorizedRecordReader;
import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
import org.apache.hadoop.hive.ql.io.orc.OrcRecordUpdater;
import org.apache.hadoop.hive.ql.io.orc.OrcSplit;
@@ -85,7 +86,8 @@
import static org.apache.hadoop.hive.llap.LlapHiveUtils.throwIfCacheOnlyRead;
-class LlapRecordReader implements RecordReader<NullWritable,
VectorizedRowBatch>, Consumer<ColumnVectorBatch> {
+final class LlapRecordReader implements RecordReader<NullWritable,
VectorizedRowBatch>, Consumer<ColumnVectorBatch>,
+ RowPositionAwareVectorizedRecordReader {
private static final Logger LOG =
LoggerFactory.getLogger(LlapRecordReader.class);
private static final Object DONE_OBJECT = new Object();
@@ -96,7 +98,6 @@ class LlapRecordReader implements RecordReader<NullWritable,
VectorizedRowBatch>
private final SearchArgument sarg;
private final VectorizedRowBatchCtx rbCtx;
private final boolean isVectorized;
- private final boolean probeDecodeEnabled;
private VectorizedOrcAcidRowBatchReader acidReader;
private Object[] partitionValues;
private VectorizedRowBatch acidInputVrb;
@@ -222,8 +223,8 @@ private LlapRecordReader(MapWork mapWork, JobConf job,
FileSplit split,
this.includes = new IncludesImpl(tableIncludedCols, isAcidFormat, rbCtx,
schema, job, isAcidScan && acidReader.includeAcidColumns());
- this.probeDecodeEnabled = HiveConf.getBoolVar(jobConf,
ConfVars.HIVE_OPTIMIZE_SCAN_PROBEDECODE);
- if (this.probeDecodeEnabled) {
+ boolean probeDecodeEnabled = HiveConf.getBoolVar(jobConf,
ConfVars.HIVE_OPTIMIZE_SCAN_PROBEDECODE);
+ if (probeDecodeEnabled) {
includes.setProbeDecodeContext(mapWork.getProbeDecodeContext());
LOG.info("LlapRecordReader ProbeDecode is enabled");
}
@@ -301,34 +302,32 @@ static int determineQueueLimit(long maxBufferedSize,
} else {
PrimitiveTypeInfo pti = (PrimitiveTypeInfo) ti;
switch (pti.getPrimitiveCategory()) {
- case BINARY:
- case CHAR:
- case VARCHAR:
- case STRING:
- colWeight = COL_WEIGHT_STRING;
- break;
+ case BINARY:
+ case CHAR:
+ case VARCHAR:
+ case STRING:
+ colWeight = COL_WEIGHT_STRING;
+ break;
//Timestamp column vector uses an int and long arrays
- case TIMESTAMP:
- case INTERVAL_DAY_TIME:
- colWeight = 2;
- break;
- case DECIMAL:
- boolean useDecimal64 = false;
- if (ti instanceof DecimalTypeInfo) {
- DecimalTypeInfo dti = (DecimalTypeInfo) ti;
- if (dti.getPrecision() <= TypeDescription.MAX_DECIMAL64_PRECISION
&& decimal64Support) {
+ case TIMESTAMP:
+ case INTERVAL_DAY_TIME:
+ colWeight = 2;
+ break;
+ case DECIMAL:
+ boolean useDecimal64 = false;
+ if (ti instanceof DecimalTypeInfo dti &&
+ dti.getPrecision() <= TypeDescription.MAX_DECIMAL64_PRECISION
&& decimal64Support) {
useDecimal64 = true;
}
- }
- // decimal_64 column vectors gets the same weight as long column
vectors
- if (useDecimal64) {
+ // decimal_64 column vectors gets the same weight as long column
vectors
+ if (useDecimal64) {
+ colWeight = 1;
+ } else {
+ colWeight = COL_WEIGHT_HIVEDECIMAL;
+ }
+ break;
+ default:
colWeight = 1;
- } else {
- colWeight = COL_WEIGHT_HIVEDECIMAL;
- }
- break;
- default:
- colWeight = 1;
}
}
totalWeight += colWeight * 8 * scale;
@@ -500,7 +499,7 @@ private AcidWrapper(VectorizedRowBatch acidVrb) {
}
@Override
- public boolean next(NullWritable key, VectorizedRowBatch value) throws
IOException {
+ public boolean next(NullWritable key, VectorizedRowBatch value) {
return true;
}
@@ -515,16 +514,16 @@ public VectorizedRowBatch createValue() {
}
@Override
- public long getPos() throws IOException {
+ public long getPos() {
return 0;
}
@Override
- public void close() throws IOException {
+ public void close() {
}
@Override
- public float getProgress() throws IOException {
+ public float getProgress() {
return 0;
}
}
@@ -581,6 +580,12 @@ ColumnVectorBatch nextCvb() throws InterruptedException,
IOException {
return lastCvb;
}
+ /** Long.MIN_VALUE (unknown) when the producer does not track row positions.
*/
+ @Override
+ public long getRowNumber() {
+ return lastCvb.getStartRowInFile();
+ }
+
@Override
public NullWritable createKey() {
return NullWritable.get();
@@ -652,7 +657,11 @@ public void setError(Throwable t) throws
InterruptedException {
private void enqueueInternal(Object o) throws InterruptedException {
// We need to loop here to handle the case where consumer goes away.
- do {} while (!isClosed && !isInterrupted && !queue.offer(o, 100,
TimeUnit.MILLISECONDS));
+ while (!isClosed && !isInterrupted) {
+ if (queue.offer(o, 100, TimeUnit.MILLISECONDS)) {
+ return;
+ }
+ }
}
@Override
@@ -665,10 +674,6 @@ void setPartitionValues(Object[] partitionValues) {
this.partitionValues = partitionValues;
}
- ReadPipeline getReadPipeline() {
- return rp;
- }
-
/** This class encapsulates include-related logic for LLAP readers. It is
not actually specific
* to LLAP IO but in LLAP IO in particular, I want to encapsulate all this
mess for now until
* we have smth better like Schema Evolution v2. This can also
hypothetically encapsulate
diff --git
a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/GenericColumnVectorProducer.java
b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/GenericColumnVectorProducer.java
index 49426eb7363..562cd973f14 100644
---
a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/GenericColumnVectorProducer.java
+++
b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/GenericColumnVectorProducer.java
@@ -21,6 +21,7 @@
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.TimeZone;
@@ -59,6 +60,7 @@
import org.apache.orc.OrcProto.RowIndex;
import org.apache.orc.OrcProto.RowIndexEntry;
import org.apache.orc.OrcProto.Type;
+import org.apache.orc.StripeInformation;
import org.apache.orc.TypeDescription;
public class GenericColumnVectorProducer implements ColumnVectorProducer {
@@ -304,5 +306,15 @@ public OrcFile.Version getFileVersion() {
public CalendarKind getCalendar() {
return CalendarKind.JULIAN_GREGORIAN;
}
+
+ @Override
+ public List<StripeInformation> getStripes() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public int getRowIndexStride() {
+ return 0;
+ }
}
}
diff --git
a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java
b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java
index 36e1ab3c54a..78e954fb1e4 100644
---
a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java
+++
b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java
@@ -70,6 +70,7 @@ public class OrcEncodedDataConsumer
private TypeReader[] columnReaders;
private int previousStripeIndex = -1;
private ConsumerFileMetadata fileMetadata; // We assume one request is only
for one file.
+ private long[] stripeStartRows; // row number in the file where each stripe
starts; null without file positions
private CompressionCodec codec;
private List<ConsumerStripeMetadata> stripes;
private SchemaEvolution evolution;
@@ -95,6 +96,12 @@ public void setUseDecimal64ColumnVectors(final boolean
useDecimal64ColumnVectors
public void setFileMetadata(ConsumerFileMetadata f) {
assert fileMetadata == null;
fileMetadata = f;
+ if (!f.getStripes().isEmpty()) {
+ stripeStartRows = new long[f.getStripes().size()];
+ for (int i = 1; i < stripeStartRows.length; ++i) {
+ stripeStartRows[i] = stripeStartRows[i - 1] + f.getStripes().get(i -
1).getNumberOfRows();
+ }
+ }
stripes = new ArrayList<>(f.getStripeCount());
codec = WriterImpl.createCodec(fileMetadata.getCompressionKind());
}
@@ -137,6 +144,13 @@ protected void decodeBatch(OrcEncodedColumnBatch batch,
}
int maxBatchesRG = (int) ((nonNullRowCount /
VectorizedRowBatch.DEFAULT_SIZE) + 1);
int batchSize = VectorizedRowBatch.DEFAULT_SIZE;
+ long rgStartRow = Long.MIN_VALUE;
+ if (stripeStartRows != null) {
+ rgStartRow = stripeStartRows[currentStripeIndex];
+ if (rgIdx != OrcEncodedColumnBatch.ALL_RGS) {
+ rgStartRow += (long) rgIdx * fileMetadata.getRowIndexStride();
+ }
+ }
TypeDescription fileSchema = fileMetadata.getSchema();
if (columnReaders == null || !sameStripe || noIndex) {
@@ -155,6 +169,8 @@ protected void decodeBatch(OrcEncodedColumnBatch batch,
ColumnVectorBatch cvb = takeBatch(batchSize);
// assert cvb.cols.length == batch.getColumnIxs().length; // Must be
constant per split.
+ cvb.setStartRowInFile(stripeStartRows == null ?
+ Long.MIN_VALUE : rgStartRow + (long) i *
VectorizedRowBatch.DEFAULT_SIZE);
for (int idx = 0; idx < columnReaders.length; ++idx) {
/*
* Currently, ORC's TreeReaderFactory class does this:
@@ -302,11 +318,15 @@ private ColumnVector createColumn(TypeDescription type,
int batchSize, final boo
private void positionInStreams(TypeReader[] columnReaders,
OrcBatchKey batchKey, ConsumerStripeMetadata stripeMetadata) throws
IOException {
PositionProvider[] pps = createPositionProviders(columnReaders, batchKey,
stripeMetadata);
- if (pps == null) return;
- for (int i = 0; i < columnReaders.length; i++) {
- if (columnReaders[i] == null) continue;
+ if (pps == null) {
+ return;
+ }
+ for (TypeReader columnReader : columnReaders) {
+ if (columnReader == null) {
+ continue;
+ }
// TODO: we could/should trace seek destinations; pps needs a "peek"
method
- columnReaders[i].seek(pps, TypeReader.ReadPhase.ALL);
+ columnReader.seek(pps, TypeReader.ReadPhase.ALL);
}
}
@@ -318,9 +338,10 @@ private void repositionInStreams(TypeReader[]
columnReaders,
if (LlapIoImpl.ORC_LOGGER.isTraceEnabled()) {
LlapIoImpl.ORC_LOGGER.trace("Created pps {}", Arrays.toString(pps));
}
- if (pps == null) return;
- for (int i = 0; i < columnReaders.length; i++) {
- TypeReader reader = columnReaders[i];
+ if (pps == null) {
+ return;
+ }
+ for (TypeReader reader : columnReaders) {
if (reader == null) continue;
// Note: we assume this never happens for SerDe reader - the batch would
never have vectors.
// That is always true now; but it wasn't some day, the below would
throw in getColumnData.
@@ -329,7 +350,7 @@ private void repositionInStreams(TypeReader[] columnReaders,
// SettableTreeReader so that we can avoid this check.
if (reader instanceof EncodedTreeReaderFactory.TimestampStreamReader &&
!sameStripe) {
((EncodedTreeReaderFactory.TimestampStreamReader) reader)
- .updateTimezone(stripeMetadata.getWriterTimezone());
+ .updateTimezone(stripeMetadata.getWriterTimezone());
}
reader.seek(pps, TypeReader.ReadPhase.ALL);
}
@@ -359,9 +380,7 @@ private PositionProvider[] createPositionProviders(
if (!stripeMetadata.supportsRowIndexes()) {
PositionProvider singleRgPp = new IndexlessPositionProvider();
pps = new PositionProvider[stripeMetadata.getEncodings().size()];
- for (int i = 0; i < pps.length; ++i) {
- pps[i] = singleRgPp;
- }
+ Arrays.fill(pps, singleRgPp);
} else {
int rowGroupIndex = batchKey.rgIx;
if (rowGroupIndex == OrcEncodedColumnBatch.ALL_RGS) {
diff --git
a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/ConsumerFileMetadata.java
b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/ConsumerFileMetadata.java
index 928b4833675..59a3ccf13e3 100644
---
a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/ConsumerFileMetadata.java
+++
b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/ConsumerFileMetadata.java
@@ -25,6 +25,7 @@
import org.apache.orc.OrcFile;
import org.apache.orc.OrcProto.CalendarKind;
import org.apache.orc.OrcProto.Type;
+import org.apache.orc.StripeInformation;
import org.apache.orc.TypeDescription;
public interface ConsumerFileMetadata {
@@ -34,4 +35,7 @@ public interface ConsumerFileMetadata {
TypeDescription getSchema() throws FileFormatException;
OrcFile.Version getFileVersion();
CalendarKind getCalendar();
+ /** Every stripe of the file in order, or empty when the data has no file
positions (re-encoded input). */
+ List<StripeInformation> getStripes();
+ int getRowIndexStride();
}
diff --git
a/llap-server/src/test/org/apache/hadoop/hive/llap/io/decode/TestOrcEncodedDataConsumerRowPositions.java
b/llap-server/src/test/org/apache/hadoop/hive/llap/io/decode/TestOrcEncodedDataConsumerRowPositions.java
new file mode 100644
index 00000000000..45c8ad6d33f
--- /dev/null
+++
b/llap-server/src/test/org/apache/hadoop/hive/llap/io/decode/TestOrcEncodedDataConsumerRowPositions.java
@@ -0,0 +1,299 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hive.llap.io.decode;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static
org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_NAME;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.llap.counters.QueryFragmentCounters;
+import org.apache.hadoop.hive.llap.io.api.LlapProxy;
+import org.apache.hadoop.hive.llap.io.api.impl.ColumnVectorBatch;
+import org.apache.hadoop.hive.llap.io.decode.ColumnVectorProducer.Includes;
+import
org.apache.hadoop.hive.llap.io.decode.GenericColumnVectorProducer.SerDeStripeMetadata;
+import org.apache.hadoop.hive.llap.io.metadata.ConsumerFileMetadata;
+import org.apache.hadoop.hive.llap.metrics.LlapDaemonIOMetrics;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx;
+import org.apache.hadoop.hive.ql.io.IOConstants;
+import org.apache.hadoop.hive.ql.io.RowPositionAwareVectorizedRecordReader;
+import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
+import org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat;
+import org.apache.hadoop.hive.ql.io.orc.encoded.Consumer;
+import org.apache.hadoop.hive.ql.io.orc.encoded.IoTrace;
+import org.apache.hadoop.hive.ql.io.orc.encoded.Reader.OrcEncodedColumnBatch;
+import org.apache.hadoop.hive.ql.plan.MapWork;
+import org.apache.hadoop.hive.ql.plan.PartitionDesc;
+import org.apache.hadoop.hive.ql.plan.TableDesc;
+import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RecordReader;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.orc.CompressionKind;
+import org.apache.orc.OrcFile;
+import org.apache.orc.OrcProto.ColumnEncoding;
+import org.apache.orc.StripeInformation;
+import org.apache.orc.TypeDescription;
+import org.apache.orc.Writer;
+import org.apache.tez.common.counters.TezCounters;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Absolute file row positions reported by {@link OrcEncodedDataConsumer} on
every emitted batch.
+ * ORC files are read end to end through LLAP IO; the position-less SerDe case
is driven straight
+ * into decodeBatch.
+ */
+public class TestOrcEncodedDataConsumerRowPositions {
+
+ private static final TypeDescription SCHEMA =
TypeDescription.fromString("struct<id:bigint,name:string>");
+ private static final int STRIDE = 2500;
+ // Neither a multiple of the row index stride nor of
VectorizedRowBatch.DEFAULT_SIZE.
+ private static final int[] STRIPE_ROWS = {8300, 6100, 3700};
+ private static final int[] NO_INDEX_STRIPE_ROWS = {5300, 2700};
+ private static final List<Integer> ALL_COLUMNS = Arrays.asList(0, 1);
+
+ private static final LlapDaemonIOMetrics IO_METRICS =
+ LlapDaemonIOMetrics.create("TestOrcEncodedDataConsumerRowPositions-io",
"1", null);
+
+ private static HiveConf daemonConf;
+ private static Path indexedFile;
+ private static Path noIndexFile;
+
+ @BeforeClass
+ public static void setUpClass() throws Exception {
+ daemonConf = new HiveConf();
+ HiveConf.setVar(daemonConf, ConfVars.LLAP_IO_MEMORY_MODE, "none");
+
+ Path tmpDir = new
Path(Files.createTempDirectory("llap-orc-row-positions").toString());
+ indexedFile = new Path(tmpDir, "indexed.orc");
+ noIndexFile = new Path(tmpDir, "no-index.orc");
+ writeFile(indexedFile, STRIDE, STRIPE_ROWS);
+ writeFile(noIndexFile, 0, NO_INDEX_STRIPE_ROWS);
+ assertLayout(indexedFile, STRIDE, STRIPE_ROWS);
+ assertLayout(noIndexFile, 0, NO_INDEX_STRIPE_ROWS);
+
+ LlapProxy.setDaemon(true);
+ LlapProxy.initializeLlapIo(daemonConf);
+ }
+
+ @AfterClass
+ public static void tearDownClass() {
+ LlapProxy.close();
+ }
+
+ @Test
+ public void testRowGroupPositionsAcrossStripes() throws Exception {
+ assertEquals(expectedBatches(STRIDE, STRIPE_ROWS),
readThroughLlap(indexedFile, ALL_COLUMNS));
+ }
+
+ @Test
+ public void testIndexDisabledFileStartsAtStripeBoundary() throws Exception {
+ assertEquals(expectedBatches(Integer.MAX_VALUE, NO_INDEX_STRIPE_ROWS),
readThroughLlap(noIndexFile, ALL_COLUMNS));
+ }
+
+ /** Without columns, the reader decodes each stripe as one ALL_RGS batch. */
+ @Test
+ public void testNoColumnReadStartsAtStripeBoundary() throws Exception {
+ assertEquals(expectedBatches(Integer.MAX_VALUE, STRIPE_ROWS),
+ readThroughLlap(indexedFile, Collections.emptyList()));
+ }
+
+ @Test
+ public void testNoStripesReportsNoPosition() throws Exception {
+ assertEquals(Collections.nCopies(3, Long.MIN_VALUE), decodeStripe(2 *
VectorizedRowBatch.DEFAULT_SIZE + 1));
+ }
+
+ // ---- LLAP IO end to end ----
+
+ /**
+ * "startRowInFile:size" per batch, in the order LLAP returned them. When
columns are read, the first row of each
+ * batch must be the row at the reported position.
+ */
+ private static List<String> readThroughLlap(Path path, List<Integer>
columnIds) throws Exception {
+ RecordReader<NullWritable, VectorizedRowBatch> reader = llapReader(path,
columnIds);
+ List<String> batches = new ArrayList<>();
+ try {
+ VectorizedRowBatch vrb = reader.createValue();
+ while (reader.next(NullWritable.get(), vrb)) {
+ long position = ((RowPositionAwareVectorizedRecordReader)
reader).getRowNumber();
+ if (!columnIds.isEmpty()) {
+ assertEquals("id of the first row of the batch at " + position,
+ position, ((LongColumnVector) vrb.cols[0]).vector[0]);
+ }
+ batches.add(position + ":" + vrb.size);
+ }
+ } finally {
+ reader.close();
+ }
+ return batches;
+ }
+
+ private static RecordReader<NullWritable, VectorizedRowBatch>
llapReader(Path path, List<Integer> columnIds)
+ throws IOException {
+ JobConf job = jobConf(path);
+ long fileLen = path.getFileSystem(job).getFileStatus(path).getLen();
+ RecordReader<NullWritable, VectorizedRowBatch> reader =
LlapProxy.getIo().orc().vectorizedReaderForPath(
+ null, path, null, columnIds, job, 0, fileLen, Reporter.NULL);
+ assertNotNull("LLAP should handle this ORC read", reader);
+ return reader;
+ }
+
+ // ---- decodeBatch without file positions ----
+
+ /** Row positions of the batches decodeBatch emits for a zero-column read of
one SerDe stripe. */
+ private static List<Long> decodeStripe(int stripeRows) throws Exception {
+ // No stripes, as for SerDe-encoded input.
+ ConsumerFileMetadata fileMetadata = mock(ConsumerFileMetadata.class);
+ when(fileMetadata.getCompressionKind()).thenReturn(CompressionKind.NONE);
+ SerDeStripeMetadata stripeMetadata = new SerDeStripeMetadata(0);
+ stripeMetadata.setRowCount(stripeRows);
+ stripeMetadata.setEncodings(Collections.singletonList(
+
ColumnEncoding.newBuilder().setKind(ColumnEncoding.Kind.DIRECT).build()));
+ Includes includes = mock(Includes.class);
+ when(includes.getBatchReaderTypes(any())).thenReturn(new
TypeDescription[0]);
+
+ QueryFragmentCounters counters = new QueryFragmentCounters(daemonConf, new
TezCounters());
+ OrcEncodedDataConsumer consumer = new OrcEncodedDataConsumer(null,
includes, counters, IO_METRICS);
+ consumer.init(null, null, new IoTrace(0, false));
+ consumer.setFileMetadata(fileMetadata);
+ consumer.setStripeMetadata(stripeMetadata);
+
+ List<Long> starts = new ArrayList<>();
+ OrcEncodedColumnBatch batch = new OrcEncodedColumnBatch();
+ batch.init(null, 0, OrcEncodedColumnBatch.ALL_RGS, 0);
+ consumer.decodeBatch(batch, new Consumer<>() {
+ @Override
+ public void consumeData(ColumnVectorBatch cvb) {
+ starts.add(cvb.getStartRowInFile());
+ }
+
+ @Override
+ public void setDone() {
+ // decodeBatch does not call it
+ }
+
+ @Override
+ public void setError(Throwable t) {
+ throw new AssertionError(t);
+ }
+ });
+ return starts;
+ }
+
+ // ---- fixture ----
+
+ private static void writeFile(Path path, int stride, int[] stripeRows)
throws IOException {
+ try (Writer writer = OrcFile.createWriter(path,
OrcFile.writerOptions(daemonConf)
+
.setSchema(SCHEMA).rowIndexStride(stride).compress(CompressionKind.NONE))) {
+ VectorizedRowBatch batch =
SCHEMA.createRowBatch(Arrays.stream(stripeRows).max().getAsInt());
+ LongColumnVector ids = (LongColumnVector) batch.cols[0];
+ BytesColumnVector names = (BytesColumnVector) batch.cols[1];
+ long row = 0;
+ for (int s = 0; s < stripeRows.length; ++s) {
+ batch.reset();
+ batch.size = stripeRows[s];
+ for (int i = 0; i < batch.size; ++i, ++row) {
+ ids.vector[i] = row;
+ names.setVal(i, ("n" + row).getBytes(UTF_8));
+ }
+ writer.addRowBatch(batch);
+ if (s < stripeRows.length - 1) {
+ writer.writeIntermediateFooter();
+ }
+ }
+ }
+ }
+
+ private static void assertLayout(Path path, int stride, int[] stripeRows)
throws IOException {
+ try (org.apache.orc.Reader reader = OrcFile.createReader(path,
OrcFile.readerOptions(daemonConf))) {
+ assertEquals(stride, reader.getRowIndexStride());
+ List<StripeInformation> stripes = reader.getStripes();
+ assertEquals(stripeRows.length, stripes.size());
+ for (int i = 0; i < stripeRows.length; ++i) {
+ assertEquals("stripe " + i, stripeRows[i],
stripes.get(i).getNumberOfRows());
+ }
+ }
+ }
+
+ private static JobConf jobConf(Path path) {
+ JobConf job = new JobConf(daemonConf);
+ HiveConf.setBoolVar(job, ConfVars.HIVE_VECTORIZATION_ENABLED, true);
+ HiveConf.setVar(job, ConfVars.PLAN, "//tmp");
+ job.set(IOConstants.COLUMNS, "id,name");
+ job.set(IOConstants.COLUMNS_TYPES, "bigint,string");
+ job.set(ColumnProjectionUtils.ORC_SCHEMA_STRING, SCHEMA.toString());
+
+ Properties tblProps = new Properties();
+ tblProps.setProperty(META_TABLE_NAME, "default.test_orc_positions");
+ TableDesc tableDesc = new TableDesc(OrcInputFormat.class,
OrcOutputFormat.class, tblProps);
+
+ MapWork mapWork = new MapWork();
+ mapWork.setVectorMode(true);
+ mapWork.setVectorizedRowBatchCtx(new VectorizedRowBatchCtx(
+ new String[] {"id", "name"},
+ new TypeInfo[] {TypeInfoFactory.longTypeInfo,
TypeInfoFactory.stringTypeInfo},
+ null, null, 0, 0, null, new String[0], null));
+ PartitionDesc partitionDesc = new PartitionDesc();
+ partitionDesc.setTableDesc(tableDesc);
+ mapWork.addPathToPartitionInfo(path.getParent(), partitionDesc);
+ Utilities.setMapWork(job, mapWork);
+ return job;
+ }
+
+ // ---- expectations ----
+
+ /** "startRowInFile:size" of every batch of a file whose stripes are read
whole. */
+ private static List<String> expectedBatches(int stride, int[] stripeRows) {
+ List<String> expected = new ArrayList<>();
+ long base = 0;
+ for (int rows : stripeRows) {
+ for (int rgStart = 0; rgStart < rows; rgStart += stride) {
+ int rgRows = Math.min(stride, rows - rgStart);
+ for (int off = 0; off < rgRows; off +=
VectorizedRowBatch.DEFAULT_SIZE) {
+ expected.add((base + rgStart + off) + ":" +
Math.min(VectorizedRowBatch.DEFAULT_SIZE, rgRows - off));
+ }
+ }
+ base += rows;
+ }
+ return expected;
+ }
+}