This is an automated email from the ASF dual-hosted git repository.
difin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new aefa4d8fffb HIVE-29841: Iceberg: Limit compaction with a filter to the
latest partition spec only (#6728)
aefa4d8fffb is described below
commit aefa4d8fffbb0ee1fabc84118448c2f42ea0f817
Author: Dmitriy Fingerman <[email protected]>
AuthorDate: Thu Sep 3 10:33:42 2026 -0400
HIVE-29841: Iceberg: Limit compaction with a filter to the latest partition
spec only (#6728)
Co-authored-by: Dmitriy Fingerman <[email protected]>
---
...major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out | 7 +++----
..._major_compaction_partition_evolution_w_id_spec_w_filter.q.out | 7 +++----
.../ql/ddl/table/storage/compact/AlterTableCompactOperation.java | 8 ++++----
3 files changed, 10 insertions(+), 12 deletions(-)
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out
index 49cff71970f..88e5686f05a 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out
@@ -357,19 +357,19 @@ Table Parameters:
compactor.threshold.target.size 1500
current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"event_id\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"event_time\",\"required\":false,\"type\":\"timestamptz\"},{\"id\":3,\"name\":\"event_src\",\"required\":false,\"type\":\"string\"}]}
current-snapshot-id #Masked#
- current-snapshot-summary
{\"manifests-created\":\"2\",\"manifests-kept\":\"6\",\"manifests-replaced\":\"1\",\"added-data-files\":\"2\",\"deleted-data-files\":\"4\",\"added-records\":\"8\",\"deleted-records\":\"8\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"4\",\"total-records\":\"26\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"8\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\
[...]
+ current-snapshot-summary
{\"manifests-created\":\"3\",\"manifests-kept\":\"6\",\"manifests-replaced\":\"2\",\"added-data-files\":\"1\",\"deleted-data-files\":\"2\",\"added-records\":\"4\",\"deleted-records\":\"4\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"26\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"10\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes
[...]
current-snapshot-timestamp-ms #Masked#
default-partition-spec
{\"spec-id\":1,\"fields\":[{\"name\":\"event_src_trunc\",\"transform\":\"truncate[3]\",\"source-id\":3,\"field-id\":1000},{\"name\":\"event_time_month\",\"transform\":\"month\",\"source-id\":2,\"field-id\":1001}]}
format-version 2
#### A masked pattern was here ####
- numFiles 8
+ numFiles 10
numPartitions 8
numRows 26
parquet.compression zstd
#### A masked pattern was here ####
rawDataSize 0
serialization.format 1
- snapshot-count 17
+ snapshot-count 16
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
totalSize #Masked#
@@ -396,4 +396,3 @@ CompactionId Database Table Partition
Type State Worker host Worker Enqueue Time
#Masked# default ice_orc event_src_trunc=AAA/event_time_month=2024-09
MAJOR refused #Masked# manual default 0 0 0 ---
#Masked# default ice_orc event_src_trunc=BBB/event_time_month=2024-07
MAJOR succeeded #Masked# manual default 0 0 0
---
#Masked# default ice_orc event_src_trunc=BBB/event_time_month=2024-08
MAJOR succeeded #Masked# manual default 0 0 0
---
-#Masked# default ice_orc --- MAJOR succeeded #Masked#
manual default 0 0 0 ---
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q.out
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q.out
index c8706124ea5..aa622d876ca 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q.out
@@ -311,19 +311,19 @@ Table Parameters:
compactor.threshold.target.size 1500
current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]}
current-snapshot-id #Masked#
- current-snapshot-summary
{\"manifests-created\":\"3\",\"manifests-kept\":\"4\",\"manifests-replaced\":\"2\",\"added-data-files\":\"4\",\"deleted-data-files\":\"4\",\"removed-position-delete-files\":\"3\",\"removed-delete-files\":\"3\",\"added-records\":\"5\",\"deleted-records\":\"8\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"removed-position-deletes\":\"3\",\"changed-partition-count\":\"5\",\"total-records\":\"10\",\"total-files-size\":\"#Masked#\",\"total-
[...]
+ current-snapshot-summary
{\"manifests-created\":\"3\",\"manifests-kept\":\"5\",\"manifests-replaced\":\"2\",\"added-data-files\":\"1\",\"deleted-data-files\":\"1\",\"removed-position-delete-files\":\"1\",\"removed-delete-files\":\"1\",\"added-records\":\"1\",\"deleted-records\":\"2\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"removed-position-deletes\":\"1\",\"changed-partition-count\":\"1\",\"total-records\":\"13\",\"total-files-size\":\"#Masked#\",\"total-
[...]
current-snapshot-timestamp-ms #Masked#
default-partition-spec
{\"spec-id\":1,\"fields\":[{\"name\":\"company_id\",\"transform\":\"identity\",\"source-id\":5,\"field-id\":1000},{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1001}]}
format-version 2
#### A masked pattern was here ####
numFiles 8
- numPartitions 4
+ numPartitions 5
numRows 10
parquet.compression zstd
#### A masked pattern was here ####
rawDataSize 0
serialization.format 1
- snapshot-count 15
+ snapshot-count 14
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
totalSize #Masked#
@@ -350,4 +350,3 @@ CompactionId Database Table Partition
Type State Worker host Worker Enqueue Time
#Masked# default ice_orc company_id=100/dept_id=2 MAJOR
succeeded #Masked# manual default 0 0 0 ---
#Masked# default ice_orc company_id=100/dept_id=3 MAJOR
succeeded #Masked# manual default 0 0 0 ---
#Masked# default ice_orc company_id=100/dept_id=4 MAJOR refused
#Masked# manual default 0 0 0 ---
-#Masked# default ice_orc --- MAJOR succeeded #Masked#
manual default 0 0 0 ---
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/compact/AlterTableCompactOperation.java
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/compact/AlterTableCompactOperation.java
index adf4592f3df..6e08d702199 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/compact/AlterTableCompactOperation.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/compact/AlterTableCompactOperation.java
@@ -135,11 +135,11 @@ else if (desc.getPartitionSpec() != null) {
compactionRequest, ServerUtils.hostname(), txnHandler,
context.getConf());
parseCompactionResponse(compactionResponse, table,
partitionMapEntry.getKey());
}
- // If Iceberg table had partition evolution, it will create compaction
request without partition specification,
- // and it will compact all files from old partition specs, besides
compacting partitions of current spec in parallel.
+ // If Iceberg table had partition evolution and the where filter is not
supplied, it will create a compaction
+ // request without partition specification, and it will compact all
files from old partition specs,
+ // besides compacting partitions of current spec in parallel.
if (DDLUtils.isIcebergTable(table) &&
table.getStorageHandler().hasUndergonePartitionEvolution(table) &&
- (desc.getFilterExpr() == null || !table.getStorageHandler()
- .getPartitionsByExpr(table, desc.getFilterExpr(),
false).isEmpty())) {
+ desc.getFilterExpr() == null) {
compactionRequest.setPartitionname(null);
CompactionResponse compactionResponse =
txnHandler.compact(compactionRequest);
parseCompactionResponse(compactionResponse, table,
compactionRequest.getPartitionname());
