This is an automated email from the ASF dual-hosted git repository.
deniskuzZ pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 96c8b2abde0 HIVE-30115: Parquet: an empty projection reads the whole
file schema (#6840)
96c8b2abde0 is described below
commit 96c8b2abde07c18b1d72275e7863951524159917
Author: Denys Kuzmenko <[email protected]>
AuthorDate: Wed Oct 28 12:57:35 2026 +0100
HIVE-30115: Parquet: an empty projection reads the whole file schema (#6840)
---
.../src/test/queries/positive/row_lineage.q | 1 +
.../src/test/results/positive/row_lineage.q.out | 12 ++++++++
.../io/parquet/read/DataWritableReadSupport.java | 4 +--
.../parquet/read/TestDataWritableReadSupport.java | 29 ++++++++++++++++-
.../queries/clientpositive/parquet_llap_counters.q | 3 ++
.../llap/parquet_llap_counters.q.out | 36 ++++++++++++++++++++++
6 files changed, 82 insertions(+), 3 deletions(-)
diff --git a/iceberg/iceberg-handler/src/test/queries/positive/row_lineage.q
b/iceberg/iceberg-handler/src/test/queries/positive/row_lineage.q
index 1d4f5480a05..9e47ca68ed3 100644
--- a/iceberg/iceberg-handler/src/test/queries/positive/row_lineage.q
+++ b/iceberg/iceberg-handler/src/test/queries/positive/row_lineage.q
@@ -5,6 +5,7 @@ select id, name, balance, ROW__LINEAGE__ID,
LAST__UPDATED__SEQUENCE__NUMBER from
update ice_t set balance = 500 where id = 2;
select id, name, balance, ROW__LINEAGE__ID, LAST__UPDATED__SEQUENCE__NUMBER
from ice_t order by id;
+select ROW__LINEAGE__ID, LAST__UPDATED__SEQUENCE__NUMBER from ice_t order by
ROW__LINEAGE__ID;
-- Test filtering with row lineage columns
select id, name, ROW__LINEAGE__ID, LAST__UPDATED__SEQUENCE__NUMBER from ice_t
where ROW__LINEAGE__ID = 1;
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/row_lineage.q.out
b/iceberg/iceberg-handler/src/test/results/positive/row_lineage.q.out
index 5275d3baab4..ec01f389c8a 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/row_lineage.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/row_lineage.q.out
@@ -48,6 +48,18 @@ POSTHOOK: Output: hdfs://### HDFS PATH ###
2 bbb 500 1 2
3 ccc 82 2 1
4 ddd 91 3 1
+PREHOOK: query: select ROW__LINEAGE__ID, LAST__UPDATED__SEQUENCE__NUMBER from
ice_t order by ROW__LINEAGE__ID
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_t
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select ROW__LINEAGE__ID, LAST__UPDATED__SEQUENCE__NUMBER from
ice_t order by ROW__LINEAGE__ID
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_t
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+0 1
+1 2
+2 1
+3 1
PREHOOK: query: select id, name, ROW__LINEAGE__ID,
LAST__UPDATED__SEQUENCE__NUMBER from ice_t where ROW__LINEAGE__ID = 1
PREHOOK: type: QUERY
PREHOOK: Input: default@ice_t
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java
index dd4eef7bbab..eae5ab5aff1 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java
@@ -435,7 +435,7 @@ public static MessageType getRequestedSchema(
List<Integer> indexColumnsWanted =
ColumnProjectionUtils.getReadColumnIDs(configuration);
//TODO Duplicated code for init method since vectorization reader path
doesn't support Nested
// column pruning so far. See HIVE-15156
- if (!ColumnProjectionUtils.isReadAllColumns(configuration) &&
!indexColumnsWanted.isEmpty()) {
+ if (!ColumnProjectionUtils.isReadAllColumns(configuration)) {
return DataWritableReadSupport
.getSchemaByIndex(tableSchema, columnNamesList, indexColumnsWanted);
} else {
@@ -468,7 +468,7 @@ private static MessageType getRequestedPrunedSchema(
Configuration configuration) {
Set<String> groupPaths =
ColumnProjectionUtils.getNestedColumnPaths(configuration);
List<Integer> indexColumnsWanted =
ColumnProjectionUtils.getReadColumnIDs(configuration);
- if (!ColumnProjectionUtils.isReadAllColumns(configuration) &&
!indexColumnsWanted.isEmpty()) {
+ if (!ColumnProjectionUtils.isReadAllColumns(configuration)) {
return getProjectedSchema(fileSchema, columnNamesList,
indexColumnsWanted, groupPaths);
} else {
return fileSchema;
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/read/TestDataWritableReadSupport.java
b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/read/TestDataWritableReadSupport.java
index a351c218b6d..6844ea1723b 100644
---
a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/read/TestDataWritableReadSupport.java
+++
b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/read/TestDataWritableReadSupport.java
@@ -19,14 +19,20 @@
package org.apache.hadoop.hive.ql.io.parquet.read;
import com.google.common.collect.Sets;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.ql.io.IOConstants;
+import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.parquet.hadoop.api.InitContext;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.MessageTypeParser;
import org.junit.Test;
import java.util.Arrays;
-import java.util.HashSet;
+import java.util.Collections;
import static
org.apache.hadoop.hive.ql.io.parquet.HiveParquetSchemaTestUtils.testConversion;
+import static org.junit.Assert.assertEquals;
public class TestDataWritableReadSupport {
@Test
@@ -116,4 +122,25 @@ public void testGetProjectedSchema5() throws Exception {
Sets.newHashSet("structCol.subStructCol", "structCol.subStructCol.b",
"structCol.subStructCol.c")).toString());
}
+
+ @Test
+ public void testNoProjectedColumnRequestsNoColumn() {
+ MessageType fileSchema = MessageTypeParser.parseMessageType("""
+ message hive_schema {
+ optional int32 a;
+ optional binary b;
+ }
+ """);
+ Configuration conf = new Configuration();
+ conf.set(IOConstants.COLUMNS, "a,b");
+ conf.set(IOConstants.COLUMNS_TYPES, "int,string");
+ conf.setBoolean(ColumnProjectionUtils.READ_ALL_COLUMNS, false);
+
+ assertEquals(0, DataWritableReadSupport.getRequestedSchema(false,
Arrays.asList("a", "b"),
+ Arrays.asList(TypeInfoFactory.intTypeInfo,
TypeInfoFactory.stringTypeInfo), fileSchema, conf)
+ .getFieldCount());
+ assertEquals(0, new DataWritableReadSupport()
+ .init(new InitContext(conf, Collections.emptyMap(), fileSchema))
+ .getRequestedSchema().getFieldCount());
+ }
}
diff --git a/ql/src/test/queries/clientpositive/parquet_llap_counters.q
b/ql/src/test/queries/clientpositive/parquet_llap_counters.q
index 2d7b346ae23..f01917abdef 100644
--- a/ql/src/test/queries/clientpositive/parquet_llap_counters.q
+++ b/ql/src/test/queries/clientpositive/parquet_llap_counters.q
@@ -92,6 +92,9 @@ set hive.llap.io.cache.only=true;
select count(*), sum(i), max(s), max(d) from parquet_llap_v2_n0 where t > -100;
set hive.llap.io.cache.only=false;
+-- No column projected: the native reader fetches no chunk and still counts
every row.
+select count(*) from parquet_llap_n0;
+
DROP TABLE parquet_llap_staging_n0;
DROP TABLE parquet_llap_n0;
DROP TABLE parquet_llap_v2_n0;
diff --git
a/ql/src/test/results/clientpositive/llap/parquet_llap_counters.q.out
b/ql/src/test/results/clientpositive/llap/parquet_llap_counters.q.out
index 8a8250e9f2f..f12b495cab5 100644
--- a/ql/src/test/results/clientpositive/llap/parquet_llap_counters.q.out
+++ b/ql/src/test/results/clientpositive/llap/parquet_llap_counters.q.out
@@ -432,6 +432,42 @@ Stage-1 INPUT COUNTERS:
INPUT_FILES_Map_1: 1
RAW_INPUT_SPLITS_Map_1: 1
2092 137368470 zach zipper 49.85
+PREHOOK: query: select count(*) from parquet_llap_n0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_llap_n0
+PREHOOK: Output: hdfs://### HDFS PATH ###
+Stage-1 FILE SYSTEM COUNTERS:
+ HDFS_BYTES_READ: 0
+ HDFS_BYTES_WRITTEN: 104
+ HDFS_READ_OPS: 2
+ HDFS_LARGE_READ_OPS: 0
+ HDFS_WRITE_OPS: 2
+Stage-1 HIVE COUNTERS:
+ CREATED_FILES: 1
+ DESERIALIZE_ERRORS: 0
+ RECORDS_IN_Map_1: 2098
+ RECORDS_OUT_0: 1
+ RECORDS_OUT_INTERMEDIATE_Map_1: 2098
+ RECORDS_OUT_INTERMEDIATE_Reducer_2: 0
+ RECORDS_OUT_OPERATOR_FS_9: 1
+ RECORDS_OUT_OPERATOR_GBY_8: 1
+ RECORDS_OUT_OPERATOR_MAP_0: 0
+ RECORDS_OUT_OPERATOR_RS_7: 2098
+ RECORDS_OUT_OPERATOR_SEL_6: 2098
+ RECORDS_OUT_OPERATOR_TS_0: 2098
+ TOTAL_TABLE_ROWS_WRITTEN: 0
+Stage-1 LLAP IO COUNTERS:
+ METADATA_CACHE_HIT: 1
+ NUM_DECODED_BATCHES: 1
+ NUM_VECTOR_BATCHES: 3
+ ROWS_EMITTED: 2098
+ SELECTED_ROWGROUPS: 1
+Stage-1 INPUT COUNTERS:
+ GROUPED_INPUT_SPLITS_Map_1: 1
+ INPUT_DIRECTORIES_Map_1: 1
+ INPUT_FILES_Map_1: 1
+ RAW_INPUT_SPLITS_Map_1: 1
+2098
PREHOOK: query: DROP TABLE parquet_llap_staging_n0
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@parquet_llap_staging_n0