This is an automated email from the ASF dual-hosted git repository.
kasakrisz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new eccaafe60b3 HIVE-29819: Fix Iceberg COW UPDATE for null identity
partition column filters (#6763)
eccaafe60b3 is described below
commit eccaafe60b38c457e0af889c2256545c320b4839
Author: kokila-19 <[email protected]>
AuthorDate: Mon Sep 14 13:45:33 2026 +0530
HIVE-29819: Fix Iceberg COW UPDATE for null identity partition column
filters (#6763)
Column pruning dropped partition columns from the TableScan while filters
above
the scan still referenced them. Keep filter-referenced partition columns in
the
Iceberg scan projection.
---
.../update_iceberg_cow_null_identity_partition.q | 35 ++
.../results/positive/iceberg_colstats_folded.q.out | 6 +-
.../positive/iceberg_colstats_granularity.q.out | 8 +-
.../positive/iceberg_insert_into_partition.q.out | 4 +-
.../iceberg_insert_overwrite_partition.q.out | 4 +-
.../iceberg_partition_pruner_cache_key.q.out | 4 +-
.../positive/iceberg_pcr_null_partition.q.out | 2 +-
...pdate_iceberg_cow_null_identity_partition.q.out | 518 +++++++++++++++++++++
.../hive/ql/optimizer/ColumnPrunerProcFactory.java | 72 +++
9 files changed, 639 insertions(+), 14 deletions(-)
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/update_iceberg_cow_null_identity_partition.q
b/iceberg/iceberg-handler/src/test/queries/positive/update_iceberg_cow_null_identity_partition.q
new file mode 100644
index 00000000000..e88b8915fd6
--- /dev/null
+++
b/iceberg/iceberg-handler/src/test/queries/positive/update_iceberg_cow_null_identity_partition.q
@@ -0,0 +1,35 @@
+-- SORT_QUERY_RESULTS
+
+set hive.explain.user=false;
+
+drop table if exists iceberg_cow_partitioned;
+
+create external table iceberg_cow_partitioned (
+ index int,
+ string_col string,
+ boolean_col boolean,
+ str_col string,
+ tinyint_col int
+) partitioned by spec(str_col, tinyint_col)
+stored by iceberg
+tblproperties ('write.update.mode'='copy-on-write');
+
+insert into iceberg_cow_partitioned partition (str_col, tinyint_col)
+ values (1, 'a', true, null, 1);
+
+explain update iceberg_cow_partitioned set str_col = 'UPDATED NULLS' where
str_col is null;
+update iceberg_cow_partitioned set str_col = 'UPDATED NULLS' where str_col is
null;
+
+select * from iceberg_cow_partitioned;
+
+-- Disable vectorization
+
+set hive.vectorized.execution.enabled=false;
+
+insert into iceberg_cow_partitioned partition (str_col, tinyint_col)
+ values (2, 'b', false, null, 2);
+
+explain update iceberg_cow_partitioned set str_col = 'UPDATED NULLS' where
str_col is null;
+update iceberg_cow_partitioned set str_col = 'UPDATED NULLS' where str_col is
null;
+
+select * from iceberg_cow_partitioned;
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_folded.q.out
b/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_folded.q.out
index f046c4e5874..2dc19a43757 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_folded.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_folded.q.out
@@ -141,12 +141,12 @@ STAGE PLANS:
TableScan
alias: ice_folded
filterExpr: ((id > 4) and (p = 'a')) (type: boolean)
- Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE
Column stats: COMPLETE
Filter Operator
predicate: (id > 4) (type: boolean)
- Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE
Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 188 Basic stats:
COMPLETE Column stats: COMPLETE
Select Operator
- Statistics: Num rows: 1 Data size: 4 Basic stats:
COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 188 Basic stats:
COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count()
minReductionHashAggr: 0.4
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_granularity.q.out
b/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_granularity.q.out
index f8711b175f1..9461d2cfbf2 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_granularity.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_granularity.q.out
@@ -71,7 +71,7 @@ STAGE PLANS:
TableScan
alias: ice_p
filterExpr: (p = 'a') (type: boolean)
- Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE
Column stats: COMPLETE
Select Operator
expressions: id (type: int), 'a' (type: string)
outputColumnNames: _col0, _col1
@@ -156,7 +156,7 @@ STAGE PLANS:
TableScan
alias: ice_p
filterExpr: (p = 'a') (type: boolean)
- Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE
Column stats: PARTIAL
+ Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE
Column stats: PARTIAL
Select Operator
expressions: id (type: int), 'a' (type: string)
outputColumnNames: _col0, _col1
@@ -198,7 +198,7 @@ STAGE PLANS:
TableScan
alias: ice_p
filterExpr: (p = 'a') (type: boolean)
- Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE
Column stats: COMPLETE
Select Operator
expressions: id (type: int), 'a' (type: string)
outputColumnNames: _col0, _col1
@@ -275,7 +275,7 @@ STAGE PLANS:
TableScan
alias: ice_p
filterExpr: (p = 'a') (type: boolean)
- Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE
Column stats: COMPLETE
Select Operator
expressions: id (type: int), 'a' (type: string)
outputColumnNames: _col0, _col1
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out
b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out
index ec615722ee6..64b1f98bcc8 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out
@@ -267,7 +267,7 @@ STAGE PLANS:
TableScan
alias: ice_parquet_int
filterExpr: (pcol = 2) (type: boolean)
- Statistics: Num rows: 2 Data size: 182 Basic stats: COMPLETE
Column stats: PARTIAL
+ Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE
Column stats: PARTIAL
Select Operator
expressions: strcol (type: string), intcol (type: int), 3
(type: int)
outputColumnNames: _col0, _col1, _col2
@@ -374,7 +374,7 @@ STAGE PLANS:
TableScan
alias: ice_parquet_int
filterExpr: (pcol = 2) (type: boolean)
- Statistics: Num rows: 2 Data size: 182 Basic stats: COMPLETE
Column stats: PARTIAL
+ Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE
Column stats: PARTIAL
Select Operator
expressions: strcol (type: string), intcol (type: int), 4
(type: int)
outputColumnNames: _col0, _col1, _col2
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out
b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out
index cb6f6d671c9..e40017d1fe5 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out
@@ -57,7 +57,7 @@ STAGE PLANS:
TableScan
alias: ice_parquet_int
filterExpr: (pcol = 2) (type: boolean)
- Statistics: Num rows: 2 Data size: 182 Basic stats: COMPLETE
Column stats: PARTIAL
+ Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE
Column stats: PARTIAL
Select Operator
expressions: strcol (type: string), intcol (type: int), 1
(type: int)
outputColumnNames: _col0, _col1, _col2
@@ -164,7 +164,7 @@ STAGE PLANS:
TableScan
alias: ice_parquet_int
filterExpr: (pcol = 2) (type: boolean)
- Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE
Column stats: PARTIAL
+ Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE
Column stats: PARTIAL
Select Operator
expressions: strcol (type: string), intcol (type: int), 1
(type: int)
outputColumnNames: _col0, _col1, _col2
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/iceberg_partition_pruner_cache_key.q.out
b/iceberg/iceberg-handler/src/test/results/positive/iceberg_partition_pruner_cache_key.q.out
index fb78a8a72b8..25fb1c3fb4c 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/iceberg_partition_pruner_cache_key.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/iceberg_partition_pruner_cache_key.q.out
@@ -272,9 +272,9 @@ STAGE PLANS:
TableScan
alias: tbl_ice_pp_key
filterExpr: (a > 2) (type: boolean)
- Statistics: Num rows: 8 Data size: 51110 Basic stats:
COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 8 Data size: 32 Basic stats: COMPLETE
Column stats: PARTIAL
Select Operator
- Statistics: Num rows: 8 Data size: 51110 Basic stats:
COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 8 Data size: 32 Basic stats:
COMPLETE Column stats: PARTIAL
Group By Operator
aggregations: count()
minReductionHashAggr: 0.99
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/iceberg_pcr_null_partition.q.out
b/iceberg/iceberg-handler/src/test/results/positive/iceberg_pcr_null_partition.q.out
index 262bae9056e..05fb89172b9 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/iceberg_pcr_null_partition.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/iceberg_pcr_null_partition.q.out
@@ -58,7 +58,7 @@ STAGE PLANS:
TableScan
alias: ice_01
filterExpr: ds is null (type: boolean)
- Statistics: Num rows: 1 Data size: 171 Basic stats: COMPLETE
Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 355 Basic stats: COMPLETE
Column stats: PARTIAL
Select Operator
expressions: key (type: string), value (type: string),
null (type: string)
outputColumnNames: _col0, _col1, _col2
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_cow_null_identity_partition.q.out
b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_cow_null_identity_partition.q.out
new file mode 100644
index 00000000000..0a4da3ba403
--- /dev/null
+++
b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_cow_null_identity_partition.q.out
@@ -0,0 +1,518 @@
+PREHOOK: query: drop table if exists iceberg_cow_partitioned
+PREHOOK: type: DROPTABLE
+PREHOOK: Output: database:default
+POSTHOOK: query: drop table if exists iceberg_cow_partitioned
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: database:default
+PREHOOK: query: create external table iceberg_cow_partitioned (
+ index int,
+ string_col string,
+ boolean_col boolean,
+ str_col string,
+ tinyint_col int
+) partitioned by spec(str_col, tinyint_col)
+stored by iceberg
+tblproperties ('write.update.mode'='copy-on-write')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@iceberg_cow_partitioned
+POSTHOOK: query: create external table iceberg_cow_partitioned (
+ index int,
+ string_col string,
+ boolean_col boolean,
+ str_col string,
+ tinyint_col int
+) partitioned by spec(str_col, tinyint_col)
+stored by iceberg
+tblproperties ('write.update.mode'='copy-on-write')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@iceberg_cow_partitioned
+PREHOOK: query: insert into iceberg_cow_partitioned partition (str_col,
tinyint_col)
+ values (1, 'a', true, null, 1)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@iceberg_cow_partitioned
+POSTHOOK: query: insert into iceberg_cow_partitioned partition (str_col,
tinyint_col)
+ values (1, 'a', true, null, 1)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@iceberg_cow_partitioned
+PREHOOK: query: explain update iceberg_cow_partitioned set str_col = 'UPDATED
NULLS' where str_col is null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@iceberg_cow_partitioned
+PREHOOK: Output: default@iceberg_cow_partitioned
+POSTHOOK: query: explain update iceberg_cow_partitioned set str_col = 'UPDATED
NULLS' where str_col is null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@iceberg_cow_partitioned
+POSTHOOK: Output: default@iceberg_cow_partitioned
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Map 1 <- Union 2 (CONTAINS)
+ Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE), Union 2
(CONTAINS)
+ Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 2 (CONTAINS)
+ Reducer 7 <- Map 5 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: iceberg_cow_partitioned
+ filterExpr: str_col is null (type: boolean)
+ Statistics: Num rows: 1 Data size: 281 Basic stats: COMPLETE
Column stats: PARTIAL
+ Select Operator
+ expressions: PARTITION__SPEC__ID (type: int),
PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type:
bigint), index (type: int), string_col (type: string), boolean_col (type:
boolean), 'UPDATED NULLS' (type: string), tinyint_col (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4,
_col5, _col6, _col7, _col8
+ Statistics: Num rows: 1 Data size: 398 Basic stats:
COMPLETE Column stats: PARTIAL
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 3 Data size: 1259 Basic stats:
COMPLETE Column stats: PARTIAL
+ table:
+ input format:
org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ output format:
org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
+ serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
+ name: default.iceberg_cow_partitioned
+ Execution mode: vectorized
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: iceberg_cow_partitioned
+ filterExpr: str_col is not null (type: boolean)
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE
Column stats: PARTIAL
+ Filter Operator
+ predicate: (FILE__PATH is not null and str_col is not
null) (type: boolean)
+ Statistics: Num rows: 1 Data size: 272 Basic stats:
COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: index (type: int), string_col (type:
string), boolean_col (type: boolean), str_col (type: string), tinyint_col
(type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint),
FILE__PATH (type: string), ROW__POSITION (type: bigint)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4,
_col5, _col6, _col7, _col8
+ Statistics: Num rows: 1 Data size: 476 Basic stats:
COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ key expressions: _col7 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col7 (type: string)
+ Statistics: Num rows: 1 Data size: 476 Basic stats:
COMPLETE Column stats: PARTIAL
+ value expressions: _col0 (type: int), _col1 (type:
string), _col2 (type: boolean), _col3 (type: string), _col4 (type: int), _col5
(type: int), _col6 (type: bigint), _col8 (type: bigint)
+ Execution mode: vectorized
+ Map 5
+ Map Operator Tree:
+ TableScan
+ alias: iceberg_cow_partitioned
+ filterExpr: str_col is null (type: boolean)
+ Statistics: Num rows: 1 Data size: 281 Basic stats: COMPLETE
Column stats: PARTIAL
+ Filter Operator
+ predicate: str_col is null (type: boolean)
+ Reduce Output Operator
+ key expressions: FILE__PATH (type: string)
+ null sort order: a
+ sort order: +
+ Map-reduce partition columns: FILE__PATH (type: string)
+ Statistics: Num rows: 1 Data size: 281 Basic stats:
COMPLETE Column stats: PARTIAL
+ value expressions: index (type: int), string_col (type:
string), boolean_col (type: boolean), tinyint_col (type: int),
PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint)
+ Filter Operator
+ predicate: (FILE__PATH is not null and str_col is null)
(type: boolean)
+ Statistics: Num rows: 1 Data size: 184 Basic stats:
COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ key expressions: FILE__PATH (type: string)
+ null sort order: a
+ sort order: +
+ Map-reduce partition columns: FILE__PATH (type: string)
+ Statistics: Num rows: 1 Data size: 184 Basic stats:
COMPLETE Column stats: PARTIAL
+ Execution mode: vectorized
+ Reducer 4
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Left Semi Join 0 to 1
+ keys:
+ 0 _col7 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5,
_col6, _col7, _col8
+ Statistics: Num rows: 1 Data size: 476 Basic stats: COMPLETE
Column stats: PARTIAL
+ Select Operator
+ expressions: _col5 (type: int), _col6 (type: bigint), _col7
(type: string), _col8 (type: bigint), _col0 (type: int), _col1 (type: string),
_col2 (type: boolean), _col3 (type: string), _col4 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5,
_col6, _col7, _col8
+ Statistics: Num rows: 1 Data size: 476 Basic stats: COMPLETE
Column stats: PARTIAL
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 3 Data size: 1259 Basic stats:
COMPLETE Column stats: PARTIAL
+ table:
+ input format:
org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ output format:
org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
+ serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
+ name: default.iceberg_cow_partitioned
+ Reducer 6
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: int), VALUE._col1 (type:
string), VALUE._col2 (type: boolean), VALUE._col4 (type: int), VALUE._col5
(type: int), VALUE._col6 (type: bigint), KEY.reducesinkkey0 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6,
_col7
+ Statistics: Num rows: 1 Data size: 293 Basic stats: COMPLETE
Column stats: PARTIAL
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col7 ASC NULLS FIRST
+ partition by: _col7
+ raw input shape:
+ window functions:
+ window function definition
+ alias: row_number_window_0
+ name: row_number
+ window function: GenericUDAFRowNumberEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ Statistics: Num rows: 1 Data size: 293 Basic stats: COMPLETE
Column stats: PARTIAL
+ Filter Operator
+ predicate: (row_number_window_0 = 1) (type: boolean)
+ Statistics: Num rows: 1 Data size: 293 Basic stats:
COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: _col5 (type: int), _col6 (type: bigint),
_col7 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type:
string), _col2 (type: boolean), null (type: string), _col4 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4,
_col5, _col6, _col7, _col8
+ Statistics: Num rows: 1 Data size: 385 Basic stats:
COMPLETE Column stats: PARTIAL
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 3 Data size: 1259 Basic stats:
COMPLETE Column stats: PARTIAL
+ table:
+ input format:
org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ output format:
org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
+ serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
+ name: default.iceberg_cow_partitioned
+ Reducer 7
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string)
+ outputColumnNames: _col7
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE
Column stats: PARTIAL
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col7: string
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col7 ASC NULLS FIRST
+ partition by: _col7
+ raw input shape:
+ window functions:
+ window function definition
+ alias: row_number_window_0
+ name: row_number
+ window function: GenericUDAFRowNumberEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE
Column stats: PARTIAL
+ Filter Operator
+ predicate: (row_number_window_0 = 1) (type: boolean)
+ Statistics: Num rows: 1 Data size: 184 Basic stats:
COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: _col7 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 184 Basic stats:
COMPLETE Column stats: PARTIAL
+ Group By Operator
+ keys: _col0 (type: string)
+ minReductionHashAggr: 0.99
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 184 Basic stats:
COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 1 Data size: 184 Basic stats:
COMPLETE Column stats: PARTIAL
+ Union 2
+ Vertex: Union 2
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: false
+ table:
+ input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
+ serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
+ name: default.iceberg_cow_partitioned
+
+ Stage: Stage-3
+ Stats Work
+ Basic Stats Work:
+
+PREHOOK: query: update iceberg_cow_partitioned set str_col = 'UPDATED NULLS'
where str_col is null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@iceberg_cow_partitioned
+PREHOOK: Output: default@iceberg_cow_partitioned
+POSTHOOK: query: update iceberg_cow_partitioned set str_col = 'UPDATED NULLS'
where str_col is null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@iceberg_cow_partitioned
+POSTHOOK: Output: default@iceberg_cow_partitioned
+PREHOOK: query: select * from iceberg_cow_partitioned
+PREHOOK: type: QUERY
+PREHOOK: Input: default@iceberg_cow_partitioned
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from iceberg_cow_partitioned
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@iceberg_cow_partitioned
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1 a true UPDATED NULLS 1
+PREHOOK: query: insert into iceberg_cow_partitioned partition (str_col,
tinyint_col)
+ values (2, 'b', false, null, 2)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@iceberg_cow_partitioned
+POSTHOOK: query: insert into iceberg_cow_partitioned partition (str_col,
tinyint_col)
+ values (2, 'b', false, null, 2)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@iceberg_cow_partitioned
+PREHOOK: query: explain update iceberg_cow_partitioned set str_col = 'UPDATED
NULLS' where str_col is null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@iceberg_cow_partitioned
+PREHOOK: Output: default@iceberg_cow_partitioned
+POSTHOOK: query: explain update iceberg_cow_partitioned set str_col = 'UPDATED
NULLS' where str_col is null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@iceberg_cow_partitioned
+POSTHOOK: Output: default@iceberg_cow_partitioned
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Map 1 <- Union 2 (CONTAINS)
+ Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE), Union 2
(CONTAINS)
+ Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 2 (CONTAINS)
+ Reducer 7 <- Map 5 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: iceberg_cow_partitioned
+ filterExpr: str_col is null (type: boolean)
+ Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE
Column stats: PARTIAL
+ Select Operator
+ expressions: PARTITION__SPEC__ID (type: int),
PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type:
bigint), index (type: int), string_col (type: string), boolean_col (type:
boolean), 'UPDATED NULLS' (type: string), tinyint_col (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4,
_col5, _col6, _col7, _col8
+ Statistics: Num rows: 1 Data size: 497 Basic stats:
COMPLETE Column stats: PARTIAL
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 3 Data size: 1565 Basic stats:
COMPLETE Column stats: PARTIAL
+ table:
+ input format:
org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ output format:
org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
+ serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
+ name: default.iceberg_cow_partitioned
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: iceberg_cow_partitioned
+ filterExpr: str_col is not null (type: boolean)
+ Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE
Column stats: PARTIAL
+ Filter Operator
+ predicate: FILE__PATH is not null (type: boolean)
+ Statistics: Num rows: 1 Data size: 380 Basic stats:
COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: index (type: int), string_col (type:
string), boolean_col (type: boolean), str_col (type: string), tinyint_col
(type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint),
FILE__PATH (type: string), ROW__POSITION (type: bigint)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4,
_col5, _col6, _col7, _col8
+ Statistics: Num rows: 1 Data size: 584 Basic stats:
COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ key expressions: _col7 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col7 (type: string)
+ Statistics: Num rows: 1 Data size: 584 Basic stats:
COMPLETE Column stats: PARTIAL
+ value expressions: _col0 (type: int), _col1 (type:
string), _col2 (type: boolean), _col3 (type: string), _col4 (type: int), _col5
(type: int), _col6 (type: bigint), _col8 (type: bigint)
+ Map 5
+ Map Operator Tree:
+ TableScan
+ alias: iceberg_cow_partitioned
+ filterExpr: str_col is null (type: boolean)
+ Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE
Column stats: PARTIAL
+ Filter Operator
+ predicate: str_col is null (type: boolean)
+ Reduce Output Operator
+ key expressions: FILE__PATH (type: string)
+ null sort order: a
+ sort order: +
+ Map-reduce partition columns: FILE__PATH (type: string)
+ Statistics: Num rows: 1 Data size: 380 Basic stats:
COMPLETE Column stats: PARTIAL
+ value expressions: index (type: int), string_col (type:
string), boolean_col (type: boolean), tinyint_col (type: int),
PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint)
+ Filter Operator
+ predicate: (FILE__PATH is not null and str_col is null)
(type: boolean)
+ Statistics: Num rows: 1 Data size: 184 Basic stats:
COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ key expressions: FILE__PATH (type: string)
+ null sort order: a
+ sort order: +
+ Map-reduce partition columns: FILE__PATH (type: string)
+ Statistics: Num rows: 1 Data size: 184 Basic stats:
COMPLETE Column stats: PARTIAL
+ Reducer 4
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Left Semi Join 0 to 1
+ keys:
+ 0 _col7 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5,
_col6, _col7, _col8
+ Statistics: Num rows: 1 Data size: 584 Basic stats: COMPLETE
Column stats: PARTIAL
+ Select Operator
+ expressions: _col5 (type: int), _col6 (type: bigint), _col7
(type: string), _col8 (type: bigint), _col0 (type: int), _col1 (type: string),
_col2 (type: boolean), _col3 (type: string), _col4 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5,
_col6, _col7, _col8
+ Statistics: Num rows: 1 Data size: 584 Basic stats: COMPLETE
Column stats: PARTIAL
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 3 Data size: 1565 Basic stats:
COMPLETE Column stats: PARTIAL
+ table:
+ input format:
org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ output format:
org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
+ serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
+ name: default.iceberg_cow_partitioned
+ Reducer 6
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: int), VALUE._col1 (type:
string), VALUE._col2 (type: boolean), VALUE._col4 (type: int), VALUE._col5
(type: int), VALUE._col6 (type: bigint), KEY.reducesinkkey0 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6,
_col7
+ Statistics: Num rows: 1 Data size: 392 Basic stats: COMPLETE
Column stats: PARTIAL
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col7 ASC NULLS FIRST
+ partition by: _col7
+ raw input shape:
+ window functions:
+ window function definition
+ alias: row_number_window_0
+ name: row_number
+ window function: GenericUDAFRowNumberEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ Statistics: Num rows: 1 Data size: 392 Basic stats: COMPLETE
Column stats: PARTIAL
+ Filter Operator
+ predicate: (row_number_window_0 = 1) (type: boolean)
+ Statistics: Num rows: 1 Data size: 392 Basic stats:
COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: _col5 (type: int), _col6 (type: bigint),
_col7 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type:
string), _col2 (type: boolean), null (type: string), _col4 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4,
_col5, _col6, _col7, _col8
+ Statistics: Num rows: 1 Data size: 484 Basic stats:
COMPLETE Column stats: PARTIAL
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 3 Data size: 1565 Basic stats:
COMPLETE Column stats: PARTIAL
+ table:
+ input format:
org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ output format:
org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
+ serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
+ name: default.iceberg_cow_partitioned
+ Reducer 7
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string)
+ outputColumnNames: _col7
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE
Column stats: PARTIAL
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col7: string
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col7 ASC NULLS FIRST
+ partition by: _col7
+ raw input shape:
+ window functions:
+ window function definition
+ alias: row_number_window_0
+ name: row_number
+ window function: GenericUDAFRowNumberEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE
Column stats: PARTIAL
+ Filter Operator
+ predicate: (row_number_window_0 = 1) (type: boolean)
+ Statistics: Num rows: 1 Data size: 184 Basic stats:
COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: _col7 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 184 Basic stats:
COMPLETE Column stats: PARTIAL
+ Group By Operator
+ keys: _col0 (type: string)
+ minReductionHashAggr: 0.99
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 184 Basic stats:
COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 1 Data size: 184 Basic stats:
COMPLETE Column stats: PARTIAL
+ Union 2
+ Vertex: Union 2
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: false
+ table:
+ input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
+ serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
+ name: default.iceberg_cow_partitioned
+
+ Stage: Stage-3
+ Stats Work
+ Basic Stats Work:
+
+PREHOOK: query: update iceberg_cow_partitioned set str_col = 'UPDATED NULLS'
where str_col is null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@iceberg_cow_partitioned
+PREHOOK: Output: default@iceberg_cow_partitioned
+POSTHOOK: query: update iceberg_cow_partitioned set str_col = 'UPDATED NULLS'
where str_col is null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@iceberg_cow_partitioned
+POSTHOOK: Output: default@iceberg_cow_partitioned
+PREHOOK: query: select * from iceberg_cow_partitioned
+PREHOOK: type: QUERY
+PREHOOK: Input: default@iceberg_cow_partitioned
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from iceberg_cow_partitioned
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@iceberg_cow_partitioned
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1 a true UPDATED NULLS 1
+2 b false UPDATED NULLS 2
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
index 8306c650ebb..6a79d2587f2 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
@@ -38,6 +38,7 @@
import org.apache.hadoop.hive.ql.exec.AbstractMapJoinOperator;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.exec.CommonJoinOperator;
+import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
import org.apache.hadoop.hive.ql.exec.FilterOperator;
import org.apache.hadoop.hive.ql.exec.GroupByOperator;
import org.apache.hadoop.hive.ql.exec.JoinOperator;
@@ -65,6 +66,7 @@
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils;
+import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.plan.GroupByDesc;
import org.apache.hadoop.hive.ql.plan.JoinDesc;
import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
@@ -496,6 +498,7 @@ public Object process(Node nd, Stack<Node> stack,
NodeProcessorCtx ctx,
}
cols = cols == null ? new ArrayList<FieldNode>() : cols;
+ cols = includePartitionColumnsFromScanFilters(scanOp, cols);
cppCtx.getPrunedColLists().put((Operator<? extends OperatorDesc>) nd,
cols);
RowSchema inputRS = scanOp.getSchema();
@@ -503,6 +506,75 @@ public Object process(Node nd, Stack<Node> stack,
NodeProcessorCtx ctx,
return null;
}
+
+ /**
+ * For storage handlers with non-native partitions (Iceberg), keep
identity partition source
+ * columns in the scan when they are referenced by pushed storage filters
or filters above the
+ * scan in the local read pipeline. Without this, filters can reference
partition columns that
+ * were pruned from the read projection.
+ */
+ private static List<FieldNode>
includePartitionColumnsFromScanFilters(TableScanOperator scanOp,
+ List<FieldNode> cols) {
+ TableScanDesc desc = scanOp.getConf();
+ Table table = desc.getTableMetadata();
+ if (table == null || !table.hasNonNativePartitionSupport()) {
+ return cols;
+ }
+
+ ExprNodeGenericFuncDesc filterExpr = desc.getFilterExpr();
+ if (filterExpr != null) {
+ cols = mergeFieldNodesWithDesc(cols, filterExpr);
+ }
+
+ if (scanOp.getChildOperators() != null) {
+ for (Operator<? extends OperatorDesc> child :
scanOp.getChildOperators()) {
+ cols = mergePartitionColumnsFromFiltersAboveScan(child, cols);
+ }
+ }
+ return cols;
+ }
+
+ /**
+ * Walks operators above a TableScan in the local read pipeline and merges
columns referenced by
+ * Filter predicates. Traversal follows passthrough operators (Select,
PTF, lateral view join,
+ * Limit), mirroring predicate pushdown boundaries in {@code
OpProcFactory.SimpleFilterPPD}.
+ * Stops at shuffle, join, grouping and sink operators.
+ */
+ private static List<FieldNode> mergePartitionColumnsFromFiltersAboveScan(
+ Operator<? extends OperatorDesc> op, List<FieldNode> cols) {
+ if (isScanPipelineBoundary(op)) {
+ return cols;
+ }
+
+ if (op instanceof FilterOperator filterOperator) {
+ cols = mergeFieldNodesWithDesc(cols,
filterOperator.getConf().getPredicate());
+ } else if (!isPassthroughOperatorAboveScan(op)) {
+ return cols;
+ }
+
+ if (op.getChildOperators() == null) {
+ return cols;
+ }
+ for (Operator<? extends OperatorDesc> child : op.getChildOperators()) {
+ cols = mergePartitionColumnsFromFiltersAboveScan(child, cols);
+ }
+ return cols;
+ }
+
+ private static boolean isPassthroughOperatorAboveScan(Operator<? extends
OperatorDesc> op) {
+ return op instanceof SelectOperator
+ || op instanceof PTFOperator
+ || op instanceof LateralViewJoinOperator
+ || op instanceof LimitOperator;
+ }
+
+ private static boolean isScanPipelineBoundary(Operator<? extends
OperatorDesc> op) {
+ return op instanceof ReduceSinkOperator
+ || op instanceof GroupByOperator
+ || op instanceof CommonJoinOperator
+ || op instanceof UnionOperator
+ || op instanceof FileSinkOperator;
+ }
}
/** Sets up needed columns for TSOP. Mainly, transfers column names from
input
