This is an automated email from the ASF dual-hosted git repository.
ayushtkn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new ad791694626 HIVE-29815: Iceberg: [V3] Support ROW LINEAGE in
Copy-On-Write DELETE operations (#6698)
ad791694626 is described below
commit ad791694626f970727b1bcff76a784dcd7fad5c9
Author: Raghav Aggarwal <[email protected]>
AuthorDate: Wed Sep 9 20:32:38 2026 +0530
HIVE-29815: Iceberg: [V3] Support ROW LINEAGE in Copy-On-Write DELETE
operations (#6698)
---
.../src/test/queries/positive/row_lineage.q | 58 +++++++-
.../src/test/results/positive/row_lineage.q.out | 155 +++++++++++++++++++++
.../parse/rewrite/CopyOnWriteDeleteRewriter.java | 9 +-
3 files changed, 220 insertions(+), 2 deletions(-)
diff --git a/iceberg/iceberg-handler/src/test/queries/positive/row_lineage.q
b/iceberg/iceberg-handler/src/test/queries/positive/row_lineage.q
index 8e44b8a5ef6..b730a92d5c0 100644
--- a/iceberg/iceberg-handler/src/test/queries/positive/row_lineage.q
+++ b/iceberg/iceberg-handler/src/test/queries/positive/row_lineage.q
@@ -77,4 +77,60 @@ WHEN MATCHED THEN
SELECT id, data, ROW__LINEAGE__ID, LAST__UPDATED__SEQUENCE__NUMBER
FROM ice_merge_cow
-ORDER BY ROW__LINEAGE__ID;
\ No newline at end of file
+ORDER BY ROW__LINEAGE__ID;
+
+-- cow delete
+CREATE TABLE ice_cow_delete (
+ id INT,
+ data STRING
+)
+STORED BY iceberg
+TBLPROPERTIES ('format-version'='3', 'write.delete.mode'='copy-on-write');
+
+INSERT INTO ice_cow_delete VALUES
+ (1, 'apple'),
+ (2, 'banana'),
+ (3, 'cherry');
+
+SELECT id, data, ROW__LINEAGE__ID, LAST__UPDATED__SEQUENCE__NUMBER
+FROM ice_cow_delete
+ORDER BY id;
+
+DELETE FROM ice_cow_delete WHERE id = 2;
+
+SELECT id, data, ROW__LINEAGE__ID, LAST__UPDATED__SEQUENCE__NUMBER
+FROM ice_cow_delete
+ORDER BY id;
+
+-- cow delete partitioned
+CREATE TABLE ice_cow_delete_part (
+ id INT,
+ data STRING
+)
+PARTITIONED BY (part STRING)
+STORED BY iceberg
+TBLPROPERTIES ('format-version'='3', 'write.delete.mode'='copy-on-write');
+
+-- Snapshot 1: Sequence 1
+INSERT INTO ice_cow_delete_part VALUES
+ (1, 'apple', 'p1'),
+ (2, 'banana', 'p1'),
+ (3, 'cherry', 'p2'),
+ (4, 'date', 'p2');
+
+-- Snapshot 2: Sequence 2 (Adding more data to partition p1 to mix sequence
numbers)
+INSERT INTO ice_cow_delete_part VALUES
+ (5, 'elderberry', 'p1');
+
+SELECT id, data, part, ROW__LINEAGE__ID, LAST__UPDATED__SEQUENCE__NUMBER
+FROM ice_cow_delete_part
+ORDER BY id;
+
+-- Snapshot 3: Delete across partitions (Affects data files in both p1 and p2)
+DELETE FROM ice_cow_delete_part WHERE id = 2 OR id = 3;
+
+-- id=1 and id=4 should retain Sequence 1
+-- id=5 should retain Sequence 2
+SELECT id, data, part, ROW__LINEAGE__ID, LAST__UPDATED__SEQUENCE__NUMBER
+FROM ice_cow_delete_part
+ORDER BY id;
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/row_lineage.q.out
b/iceberg/iceberg-handler/src/test/results/positive/row_lineage.q.out
index 046576fa9bf..59308a02872 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/row_lineage.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/row_lineage.q.out
@@ -314,3 +314,158 @@ POSTHOOK: Output: hdfs://### HDFS PATH ###
1 a 0 1
2 bb_changed 1 2
3 c 2 1
+PREHOOK: query: CREATE TABLE ice_cow_delete (
+ id INT,
+ data STRING
+)
+STORED BY iceberg
+TBLPROPERTIES ('format-version'='3', 'write.delete.mode'='copy-on-write')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ice_cow_delete
+POSTHOOK: query: CREATE TABLE ice_cow_delete (
+ id INT,
+ data STRING
+)
+STORED BY iceberg
+TBLPROPERTIES ('format-version'='3', 'write.delete.mode'='copy-on-write')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ice_cow_delete
+PREHOOK: query: INSERT INTO ice_cow_delete VALUES
+ (1, 'apple'),
+ (2, 'banana'),
+ (3, 'cherry')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_cow_delete
+POSTHOOK: query: INSERT INTO ice_cow_delete VALUES
+ (1, 'apple'),
+ (2, 'banana'),
+ (3, 'cherry')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_cow_delete
+PREHOOK: query: SELECT id, data, ROW__LINEAGE__ID,
LAST__UPDATED__SEQUENCE__NUMBER
+FROM ice_cow_delete
+ORDER BY id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_cow_delete
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT id, data, ROW__LINEAGE__ID,
LAST__UPDATED__SEQUENCE__NUMBER
+FROM ice_cow_delete
+ORDER BY id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_cow_delete
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1 apple 0 1
+2 banana 1 1
+3 cherry 2 1
+PREHOOK: query: DELETE FROM ice_cow_delete WHERE id = 2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_cow_delete
+PREHOOK: Output: default@ice_cow_delete
+POSTHOOK: query: DELETE FROM ice_cow_delete WHERE id = 2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_cow_delete
+POSTHOOK: Output: default@ice_cow_delete
+PREHOOK: query: SELECT id, data, ROW__LINEAGE__ID,
LAST__UPDATED__SEQUENCE__NUMBER
+FROM ice_cow_delete
+ORDER BY id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_cow_delete
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT id, data, ROW__LINEAGE__ID,
LAST__UPDATED__SEQUENCE__NUMBER
+FROM ice_cow_delete
+ORDER BY id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_cow_delete
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1 apple 0 1
+3 cherry 2 1
+PREHOOK: query: CREATE TABLE ice_cow_delete_part (
+ id INT,
+ data STRING
+)
+PARTITIONED BY (part STRING)
+STORED BY iceberg
+TBLPROPERTIES ('format-version'='3', 'write.delete.mode'='copy-on-write')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ice_cow_delete_part
+POSTHOOK: query: CREATE TABLE ice_cow_delete_part (
+ id INT,
+ data STRING
+)
+PARTITIONED BY (part STRING)
+STORED BY iceberg
+TBLPROPERTIES ('format-version'='3', 'write.delete.mode'='copy-on-write')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ice_cow_delete_part
+PREHOOK: query: INSERT INTO ice_cow_delete_part VALUES
+ (1, 'apple', 'p1'),
+ (2, 'banana', 'p1'),
+ (3, 'cherry', 'p2'),
+ (4, 'date', 'p2')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_cow_delete_part
+POSTHOOK: query: INSERT INTO ice_cow_delete_part VALUES
+ (1, 'apple', 'p1'),
+ (2, 'banana', 'p1'),
+ (3, 'cherry', 'p2'),
+ (4, 'date', 'p2')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_cow_delete_part
+PREHOOK: query: INSERT INTO ice_cow_delete_part VALUES
+ (5, 'elderberry', 'p1')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_cow_delete_part
+POSTHOOK: query: INSERT INTO ice_cow_delete_part VALUES
+ (5, 'elderberry', 'p1')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_cow_delete_part
+PREHOOK: query: SELECT id, data, part, ROW__LINEAGE__ID,
LAST__UPDATED__SEQUENCE__NUMBER
+FROM ice_cow_delete_part
+ORDER BY id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_cow_delete_part
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT id, data, part, ROW__LINEAGE__ID,
LAST__UPDATED__SEQUENCE__NUMBER
+FROM ice_cow_delete_part
+ORDER BY id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_cow_delete_part
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1 apple p1 0 1
+2 banana p1 1 1
+3 cherry p2 2 1
+4 date p2 3 1
+5 elderberry p1 4 2
+PREHOOK: query: DELETE FROM ice_cow_delete_part WHERE id = 2 OR id = 3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_cow_delete_part
+PREHOOK: Output: default@ice_cow_delete_part
+POSTHOOK: query: DELETE FROM ice_cow_delete_part WHERE id = 2 OR id = 3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_cow_delete_part
+POSTHOOK: Output: default@ice_cow_delete_part
+PREHOOK: query: SELECT id, data, part, ROW__LINEAGE__ID,
LAST__UPDATED__SEQUENCE__NUMBER
+FROM ice_cow_delete_part
+ORDER BY id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_cow_delete_part
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT id, data, part, ROW__LINEAGE__ID,
LAST__UPDATED__SEQUENCE__NUMBER
+FROM ice_cow_delete_part
+ORDER BY id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_cow_delete_part
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1 apple p1 0 1
+4 date p2 3 1
+5 elderberry p1 4 2
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/CopyOnWriteDeleteRewriter.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/CopyOnWriteDeleteRewriter.java
index 73bd1db4e6c..22e06234b52 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/CopyOnWriteDeleteRewriter.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/CopyOnWriteDeleteRewriter.java
@@ -30,6 +30,9 @@
import org.apache.hadoop.hive.ql.parse.rewrite.sql.MultiInsertSqlGenerator;
import org.apache.hadoop.hive.ql.parse.rewrite.sql.SqlGeneratorFactory;
+import static
org.apache.hadoop.hive.ql.metadata.RowLineageUtils.addSourceColumnsForRowLineage;
+import static
org.apache.hadoop.hive.ql.metadata.RowLineageUtils.supportsRowLineage;
+
public class CopyOnWriteDeleteRewriter implements Rewriter<DeleteStatement> {
private final HiveConf conf;
@@ -46,6 +49,8 @@ public CopyOnWriteDeleteRewriter(HiveConf conf,
SqlGeneratorFactory sqlGenerator
public ParseUtils.ReparseResult rewrite(Context context, DeleteStatement
deleteBlock)
throws SemanticException {
+ boolean isRowLineageSupported =
supportsRowLineage(deleteBlock.getTargetTable());
+
ASTNode whereTree = deleteBlock.getWhereTree();
String whereClause = "true";
if (whereTree != null) {
@@ -57,7 +62,8 @@ public ParseUtils.ReparseResult rewrite(Context context,
DeleteStatement deleteB
MultiInsertSqlGenerator sqlGenerator =
sqlGeneratorFactory.createSqlGenerator();
- cowWithClauseBuilder.appendWith(sqlGenerator, filePathCol, whereClause);
+ cowWithClauseBuilder.appendWith(
+ sqlGenerator, null, filePathCol, whereClause, true,
isRowLineageSupported, "");
sqlGenerator.append("insert into table ");
sqlGenerator.append(sqlGenerator.getTargetTableFullName());
@@ -66,6 +72,7 @@ public ParseUtils.ReparseResult rewrite(Context context,
DeleteStatement deleteB
sqlGenerator.append(" select ");
sqlGenerator.appendAcidSelectColumns(Context.Operation.DELETE);
sqlGenerator.removeLastChar();
+ addSourceColumnsForRowLineage(isRowLineageSupported, sqlGenerator, "",
conf);
sqlGenerator.append(" from ");
sqlGenerator.append(sqlGenerator.getTargetTableFullName());
