This is an automated email from the ASF dual-hosted git repository.
difin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new ecb15540c3f HIVE-29853: Iceberg CompactionEvaluator double-counts
input files (#6737)
ecb15540c3f is described below
commit ecb15540c3fb8f49b08d6d40976f6d951a6a953d
Author: Raghav Aggarwal <[email protected]>
AuthorDate: Thu Oct 15 04:00:31 2026 +0530
HIVE-29853: Iceberg CompactionEvaluator double-counts input files (#6737)
---
.../compaction/evaluator/CompactionEvaluator.java | 1 -
.../positive/iceberg_compaction_colstats_compute.q | 4 +-
.../iceberg_major_compaction_partition_evolution.q | 5 +
...iceberg_major_compaction_partition_evolution2.q | 25 ++-
..._major_compaction_partition_evolution_ordered.q | 5 +
...ction_partition_evolution_w_dyn_spec_w_filter.q | 65 ++++---
.../iceberg_major_compaction_query_metadata.q | 17 +-
.../iceberg_minor_compaction_partition_evolution.q | 6 +
.../iceberg_minor_compaction_unpartitioned.q | 12 +-
.../positive/iceberg_row_lineage_compactions.q | 2 +
.../llap/iceberg_compaction_colstats_compute.q.out | 8 +-
...berg_major_compaction_partition_evolution.q.out | 140 +++++++++----
...erg_major_compaction_partition_evolution2.q.out | 206 ++++++++++++++++----
...or_compaction_partition_evolution_ordered.q.out | 64 +++++-
...n_partition_evolution_w_dyn_spec_w_filter.q.out | 216 +++++++++++++++------
.../iceberg_major_compaction_query_metadata.q.out | 68 +++++--
...berg_minor_compaction_partition_evolution.q.out | 100 ++++++++--
.../iceberg_minor_compaction_unpartitioned.q.out | 62 ++++--
.../llap/iceberg_row_lineage_compactions.q.out | 24 ++-
19 files changed, 773 insertions(+), 257 deletions(-)
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/evaluator/CompactionEvaluator.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/evaluator/CompactionEvaluator.java
index e3b64c39e19..f05c155c34e 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/evaluator/CompactionEvaluator.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/evaluator/CompactionEvaluator.java
@@ -73,7 +73,6 @@ public CompactionEvaluator(Table table, CompactionInfo ci,
Map<String, String> p
);
this.table = table;
this.ci = ci;
- addFiles();
}
public boolean isEligibleForCompaction() {
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_compaction_colstats_compute.q
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_compaction_colstats_compute.q
index 8fa963997b7..d79ab101185 100644
---
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_compaction_colstats_compute.q
+++
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_compaction_colstats_compute.q
@@ -11,7 +11,7 @@ set hive.iceberg.stats.collect.partlevel=true;
-- describes the whole of it
create external table ice_comp_unpart (id bigint, p string)
stored by iceberg stored as orc
-tblproperties ('format-version'='2', 'compactor.threshold.target.size'='1500');
+tblproperties ('format-version'='2', 'compactor.threshold.target.size'='1000');
insert into ice_comp_unpart values (1, 'a');
insert into ice_comp_unpart values (2, 'a');
@@ -33,7 +33,7 @@ drop table ice_comp_unpart;
create external table ice_comp (id bigint, p string)
partitioned by spec (p)
stored by iceberg stored as orc
-tblproperties ('format-version'='2', 'compactor.threshold.target.size'='1500',
+tblproperties ('format-version'='2', 'compactor.threshold.target.size'='1000',
-- a compaction runs long after the session that queued it, so the
granularity it keeps
-- statistics at is asked for the way the compactor takes any of its
settings
'compactor.hive.iceberg.stats.collect.partlevel'='true');
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution.q
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution.q
index a38a856f643..10c17d7eb42 100644
---
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution.q
+++
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution.q
@@ -42,6 +42,11 @@ alter table ice_orc create tag v1;
alter table ice_orc set partition spec(company_id, dept_id);
insert into ice_orc VALUES ('fn4','ln4', 1, 11, 100);
+insert into ice_orc VALUES ('fn41','ln41', 1, 11, 100);
+insert into ice_orc VALUES ('fn42','ln42', 1, 11, 100);
+insert into ice_orc VALUES ('fn43','ln43', 1, 11, 100);
+insert into ice_orc VALUES ('fn44','ln44', 1, 11, 100);
+insert into ice_orc VALUES ('fn45','ln45', 1, 11, 100);
insert into ice_orc VALUES ('fn5','ln5', 2, 20, 100);
insert into ice_orc VALUES ('fn6','ln6', 2, 20, 100);
insert into ice_orc VALUES (null,null, null, null, null);
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution2.q
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution2.q
index 4ca441ff0b1..faa3933d728 100644
---
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution2.q
+++
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution2.q
@@ -38,15 +38,21 @@ insert into ice_orc VALUES ('fn1','ln1', 1);
insert into ice_orc VALUES ('fn2','ln2', 1);
insert into ice_orc VALUES ('fn3','ln3', 1);
insert into ice_orc VALUES ('fn4','ln4', 1);
-delete from ice_orc where last_name in ('ln3', 'ln4');
+insert into ice_orc VALUES ('fn5','ln5', 1);
+insert into ice_orc VALUES ('fn6','ln6', 1);
+insert into ice_orc VALUES ('fn7','ln7', 1);
+delete from ice_orc where last_name in ('ln6', 'ln7');
alter table ice_orc set partition spec(dept_id);
-insert into ice_orc PARTITION(dept_id=2) VALUES ('fn5','ln5');
-insert into ice_orc PARTITION(dept_id=2) VALUES ('fn6','ln6');
-insert into ice_orc PARTITION(dept_id=2) VALUES ('fn7','ln7');
insert into ice_orc PARTITION(dept_id=2) VALUES ('fn8','ln8');
-delete from ice_orc where last_name in ('ln7', 'ln8');
+insert into ice_orc PARTITION(dept_id=2) VALUES ('fn9','ln9');
+insert into ice_orc PARTITION(dept_id=2) VALUES ('fn10','ln10');
+insert into ice_orc PARTITION(dept_id=2) VALUES ('fn11','ln11');
+insert into ice_orc PARTITION(dept_id=2) VALUES ('fn12','ln12');
+insert into ice_orc PARTITION(dept_id=2) VALUES ('fn13','ln13');
+insert into ice_orc PARTITION(dept_id=2) VALUES ('fn14','ln14');
+delete from ice_orc where last_name in ('ln13', 'ln14');
select * from ice_orc;
describe formatted ice_orc;
@@ -60,8 +66,13 @@ show compactions order by 'partition';
alter table ice_orc set partition spec();
-insert into ice_orc VALUES ('fn9','ln9', 3);
-insert into ice_orc VALUES ('fn10','ln10', 3);
+insert into ice_orc VALUES ('fn15','ln15', 3);
+insert into ice_orc VALUES ('fn16','ln16', 3);
+insert into ice_orc VALUES ('fn17','ln17', 3);
+insert into ice_orc VALUES ('fn18','ln18', 3);
+insert into ice_orc VALUES ('fn19','ln19', 3);
+insert into ice_orc VALUES ('fn20','ln20', 3);
+insert into ice_orc VALUES ('fn21','ln21', 3);
select * from ice_orc;
describe formatted ice_orc;
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_ordered.q
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_ordered.q
index 0dfe2814042..852d978cd87 100644
---
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_ordered.q
+++
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_ordered.q
@@ -49,6 +49,11 @@ insert into ice_orc VALUES
insert into ice_orc VALUES
('fn7','ln7', 2, 20, 101),
('fn8','ln8', 2, 20, 101);
+insert into ice_orc VALUES ('fn81','ln81', 2, 20, 101);
+insert into ice_orc VALUES ('fn82','ln82', 2, 20, 101);
+insert into ice_orc VALUES ('fn83','ln83', 2, 20, 101);
+insert into ice_orc VALUES ('fn84','ln84', 2, 20, 101);
+insert into ice_orc VALUES ('fn85','ln85', 2, 20, 101);
insert into ice_orc VALUES
('fn9', 'ln9', 3, 20, 101),
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q
index 896f07d5bda..830c4970c83 100644
---
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q
+++
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q
@@ -33,47 +33,60 @@ partitioned by spec(truncate(3, event_src))
stored by iceberg stored as orc
tblproperties ('compactor.threshold.target.size'='1500');
-insert into ice_orc values
- (1, cast('2023-07-20 00:00:00' as timestamp with local time zone),
'AAA_1'),
+insert into ice_orc values
+ (1, cast('2023-07-20 00:00:00' as timestamp with local time zone),
'AAA_1');
+insert into ice_orc values
(2, cast('2023-07-21 00:00:00' as timestamp with local time zone),
'AAA_2');
-insert into ice_orc values
- (3, cast('2023-07-23 00:00:00' as timestamp with local time zone),
'AAA_1'),
+insert into ice_orc values
+ (3, cast('2023-07-23 00:00:00' as timestamp with local time zone),
'AAA_1');
+insert into ice_orc values
(4, cast('2023-07-24 00:00:00' as timestamp with local time zone),
'AAA_2');
-insert into ice_orc values
- (5, cast('2023-08-04 00:00:00' as timestamp with local time zone),
'BBB_1'),
+insert into ice_orc values
+ (5, cast('2023-08-04 00:00:00' as timestamp with local time zone),
'BBB_1');
+insert into ice_orc values
(6, cast('2023-08-05 00:00:00' as timestamp with local time zone),
'BBB_2');
-insert into ice_orc values
- (7, cast('2023-08-06 00:00:00' as timestamp with local time zone),
'BBB_1'),
+insert into ice_orc values
+ (7, cast('2023-08-06 00:00:00' as timestamp with local time zone),
'BBB_1');
+insert into ice_orc values
(8, cast('2023-08-27 00:00:00' as timestamp with local time zone),
'BBB_2');
alter table ice_orc set partition spec(truncate(3, event_src),
month(event_time));
-insert into ice_orc values
- (9, cast('2024-07-20 00:00:00' as timestamp with local time zone),
'AAA_1'),
+insert into ice_orc values
+ (9, cast('2024-07-20 00:00:00' as timestamp with local time zone),
'AAA_1');
+insert into ice_orc values
(10, cast('2024-07-21 00:00:00' as timestamp with local time zone),
'AAA_2');
-insert into ice_orc values
- (11, cast('2024-08-22 00:00:00' as timestamp with local time zone),
'AAA_1'),
+insert into ice_orc values
+ (11, cast('2024-08-22 00:00:00' as timestamp with local time zone),
'AAA_1');
+insert into ice_orc values
(12, cast('2024-08-23 00:00:00' as timestamp with local time zone),
'AAA_2');
-insert into ice_orc values
- (13, cast('2024-08-24 00:00:00' as timestamp with local time zone),
'AAA_1'),
+insert into ice_orc values
+ (13, cast('2024-08-24 00:00:00' as timestamp with local time zone),
'AAA_1');
+insert into ice_orc values
(14, cast('2024-08-25 00:00:00' as timestamp with local time zone),
'AAA_2');
-insert into ice_orc values
- (15, cast('2024-09-24 00:00:00' as timestamp with local time zone),
'AAA_1'),
+insert into ice_orc values
+ (15, cast('2024-09-24 00:00:00' as timestamp with local time zone),
'AAA_1');
+insert into ice_orc values
(16, cast('2024-09-25 00:00:00' as timestamp with local time zone),
'AAA_2');
-insert into ice_orc values
- (17, cast('2024-07-01 00:00:00' as timestamp with local time zone),
'BBB_1'),
+insert into ice_orc values
+ (17, cast('2024-07-01 00:00:00' as timestamp with local time zone),
'BBB_1');
+insert into ice_orc values
(18, cast('2024-07-02 00:00:00' as timestamp with local time zone),
'BBB_2');
-insert into ice_orc values
- (19, cast('2024-07-03 00:00:00' as timestamp with local time zone),
'BBB_1'),
+insert into ice_orc values
+ (19, cast('2024-07-03 00:00:00' as timestamp with local time zone),
'BBB_1');
+insert into ice_orc values
(20, cast('2024-07-04 00:00:00' as timestamp with local time zone),
'BBB_2');
-insert into ice_orc values
- (21, cast('2024-08-03 00:00:00' as timestamp with local time zone),
'BBB_1'),
+insert into ice_orc values
+ (21, cast('2024-08-03 00:00:00' as timestamp with local time zone),
'BBB_1');
+insert into ice_orc values
(22, cast('2024-08-04 00:00:00' as timestamp with local time zone),
'BBB_2');
-insert into ice_orc values
- (23, cast('2024-08-05 00:00:00' as timestamp with local time zone),
'BBB_1'),
+insert into ice_orc values
+ (23, cast('2024-08-05 00:00:00' as timestamp with local time zone),
'BBB_1');
+insert into ice_orc values
(24, cast('2024-08-06 00:00:00' as timestamp with local time zone),
'BBB_2');
insert into ice_orc values
- (25, cast('2024-09-05 00:00:00' as timestamp with local time zone),
'BBB_1'),
+ (25, cast('2024-09-05 00:00:00' as timestamp with local time zone),
'BBB_1');
+insert into ice_orc values
(26, cast('2024-09-06 00:00:00' as timestamp with local time zone),
'BBB_2');
select * from ice_orc;
@@ -86,4 +99,4 @@ where (event_src in ('BBB_1', 'BBB_2') and event_time <
'2024-09-01 00:00:00') o
select * from ice_orc;
describe formatted ice_orc;
-show compactions order by 'partition';
\ No newline at end of file
+show compactions order by 'partition';
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_query_metadata.q
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_query_metadata.q
index ecdeacffbbf..24b23769a36 100644
---
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_query_metadata.q
+++
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_query_metadata.q
@@ -24,16 +24,13 @@ CREATE TABLE x (name VARCHAR(50), age TINYINT, num_clicks
BIGINT)
stored by iceberg stored as orc
TBLPROPERTIES ('external.table.purge'='true', 'format-version'='2',
'compactor.threshold.target.size'='1500');
-insert into x values
-('amy', 35, 123412344),
-('adxfvy', 36, 123412534),
-('amsdfyy', 37, 123417234),
-('asafmy', 38, 123412534);
-
-insert into x values
-('amerqwy', 39, 123441234),
-('amyxzcv', 40, 123341234),
-('erweramy', 45, 122341234);
+insert into x values ('amy', 35, 123412344);
+insert into x values ('adxfvy', 36, 123412534);
+insert into x values ('amsdfyy', 37, 123417234);
+insert into x values ('asafmy', 38, 123412534);
+insert into x values ('amerqwy', 39, 123441234);
+insert into x values ('amyxzcv', 40, 123341234);
+insert into x values ('erweramy', 45, 122341234);
select * from default.x.data_files;
select count(*) from default.x.data_files;
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_minor_compaction_partition_evolution.q
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_minor_compaction_partition_evolution.q
index f89fbf519e0..313103da1b2 100644
---
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_minor_compaction_partition_evolution.q
+++
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_minor_compaction_partition_evolution.q
@@ -38,6 +38,8 @@ tblproperties ('format-version'='2');
insert into ice_orc VALUES ('fn2','ln2', 1), ('fn1','ln1', 1);
insert into ice_orc VALUES ('fn4','ln4', 1), ('fn3','ln3', 1);
+insert into ice_orc VALUES ('fn15','ln15', 1);
+insert into ice_orc VALUES ('fn16','ln16', 1);
delete from ice_orc where last_name in ('ln4');
@@ -45,6 +47,8 @@ alter table ice_orc set partition spec(dept_id);
insert into ice_orc PARTITION(dept_id=2) VALUES ('fn6','ln6'), ('fn5','ln5');
insert into ice_orc PARTITION(dept_id=2) VALUES ('fn8','ln8'), ('fn7','ln7');
+insert into ice_orc PARTITION(dept_id=2) VALUES ('fn17','ln17');
+insert into ice_orc PARTITION(dept_id=2) VALUES ('fn18','ln18');
delete from ice_orc where last_name in ('ln8');
@@ -73,6 +77,8 @@ alter table ice_orc set partition spec();
insert into ice_orc VALUES ('fn9','ln9', 3), ('fn10','ln10', 3);
insert into ice_orc VALUES ('fn11','ln11', 3), ('fn12','ln12', 3);
+insert into ice_orc VALUES ('fn19','ln19', 3);
+insert into ice_orc VALUES ('fn20','ln20', 3);
select * from ice_orc;
describe formatted ice_orc;
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_minor_compaction_unpartitioned.q
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_minor_compaction_unpartitioned.q
index 61e3a82525d..3f7570c0310 100644
---
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_minor_compaction_unpartitioned.q
+++
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_minor_compaction_unpartitioned.q
@@ -33,11 +33,15 @@ stored by iceberg stored as orc
tblproperties ('format-version'='2');
insert into ice_orc VALUES ('fn1','ln1');
-insert into ice_orc VALUES ('fn5','ln5'), ('fn7','ln7'), ('fn4','ln4'),
('fn6','ln6');
-insert into ice_orc VALUES ('fn3','ln3'), ('fn2','ln2');
+insert into ice_orc VALUES ('fn5','ln5');
+insert into ice_orc VALUES ('fn7','ln7');
+insert into ice_orc VALUES ('fn4','ln4');
+insert into ice_orc VALUES ('fn6','ln6');
+insert into ice_orc VALUES ('fn3','ln3');
+insert into ice_orc VALUES ('fn2','ln2');
--- File size threshold = fragment size = target_size/fragment_ratio = 2920/8 =
365.
-alter table ice_orc set tblproperties
('compactor.threshold.target.size'='2920',
'compactor.threshold.min.input.files'='2');
+-- File size threshold = fragment size = target_size/fragment_ratio = 4000/8 =
365.
+alter table ice_orc set tblproperties
('compactor.threshold.target.size'='4000',
'compactor.threshold.min.input.files'='2');
explain alter table ice_orc COMPACT 'minor' and wait pool 'iceberg';
describe formatted ice_orc;
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_row_lineage_compactions.q
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_row_lineage_compactions.q
index 8f802d75b39..5a7de18b0a0 100644
---
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_row_lineage_compactions.q
+++
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_row_lineage_compactions.q
@@ -74,6 +74,8 @@ alter table part_tbl set tblproperties
('compactor.threshold.target.size'='8000'
alter table part_tbl set partition spec(dept_id, id);
insert into part_tbl values (5,'p5', 10);
+insert into part_tbl values (5,'p5', 10);
+insert into part_tbl values (6,'p6', 20);
insert into part_tbl values (6,'p6', 20);
SELECT *, ROW__LINEAGE__ID, LAST__UPDATED__SEQUENCE__NUMBER
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_compaction_colstats_compute.q.out
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_compaction_colstats_compute.q.out
index af2e3e249bd..87513dab0ec 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_compaction_colstats_compute.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_compaction_colstats_compute.q.out
@@ -1,12 +1,12 @@
PREHOOK: query: create external table ice_comp_unpart (id bigint, p string)
stored by iceberg stored as orc
-tblproperties ('format-version'='2', 'compactor.threshold.target.size'='1500')
+tblproperties ('format-version'='2', 'compactor.threshold.target.size'='1000')
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@ice_comp_unpart
POSTHOOK: query: create external table ice_comp_unpart (id bigint, p string)
stored by iceberg stored as orc
-tblproperties ('format-version'='2', 'compactor.threshold.target.size'='1500')
+tblproperties ('format-version'='2', 'compactor.threshold.target.size'='1000')
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@ice_comp_unpart
@@ -164,7 +164,7 @@ POSTHOOK: Output: default@ice_comp_unpart
PREHOOK: query: create external table ice_comp (id bigint, p string)
partitioned by spec (p)
stored by iceberg stored as orc
-tblproperties ('format-version'='2', 'compactor.threshold.target.size'='1500',
+tblproperties ('format-version'='2', 'compactor.threshold.target.size'='1000',
-- a compaction runs long after the session that queued it, so the
granularity it keeps
-- statistics at is asked for the way the compactor takes any of its
settings
'compactor.hive.iceberg.stats.collect.partlevel'='true')
@@ -174,7 +174,7 @@ PREHOOK: Output: default@ice_comp
POSTHOOK: query: create external table ice_comp (id bigint, p string)
partitioned by spec (p)
stored by iceberg stored as orc
-tblproperties ('format-version'='2', 'compactor.threshold.target.size'='1500',
+tblproperties ('format-version'='2', 'compactor.threshold.target.size'='1000',
-- a compaction runs long after the session that queued it, so the
granularity it keeps
-- statistics at is asked for the way the compactor takes any of its
settings
'compactor.hive.iceberg.stats.collect.partlevel'='true')
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out
index 13889cdb0e7..f4899002537 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out
@@ -75,6 +75,46 @@ POSTHOOK: query: insert into ice_orc VALUES ('fn4','ln4', 1,
11, 100)
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc VALUES ('fn41','ln41', 1, 11, 100)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc VALUES ('fn41','ln41', 1, 11, 100)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc VALUES ('fn42','ln42', 1, 11, 100)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc VALUES ('fn42','ln42', 1, 11, 100)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc VALUES ('fn43','ln43', 1, 11, 100)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc VALUES ('fn43','ln43', 1, 11, 100)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc VALUES ('fn44','ln44', 1, 11, 100)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc VALUES ('fn44','ln44', 1, 11, 100)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc VALUES ('fn45','ln45', 1, 11, 100)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc VALUES ('fn45','ln45', 1, 11, 100)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
PREHOOK: query: insert into ice_orc VALUES ('fn5','ln5', 2, 20, 100)
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
@@ -259,6 +299,11 @@ NULL NULL NULL NULL NULL
fn2 ln2a 1 10 100
fn3 ln3a 1 11 100
fn4 ln4a 1 11 100
+fn41 ln41 1 11 100
+fn42 ln42 1 11 100
+fn43 ln43 1 11 100
+fn44 ln44 1 11 100
+fn45 ln45 1 11 100
fn5 ln5a 2 20 100
fn6 ln6a 2 20 100
fn7 ln7a 2 21 100
@@ -296,19 +341,19 @@ Table Parameters:
compactor.threshold.target.size 1500
current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]}
current-snapshot-id #Masked#
- current-snapshot-summary
{\"manifests-created\":\"2\",\"manifests-kept\":\"5\",\"manifests-replaced\":\"6\",\"deleted-data-files\":\"2\",\"deleted-records\":\"2\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"2\",\"total-records\":\"17\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"17\",\"total-delete-files\":\"8\",\"total-position-deletes\":\"8\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"}
+ current-snapshot-summary
{\"manifests-created\":\"2\",\"manifests-kept\":\"5\",\"manifests-replaced\":\"6\",\"deleted-data-files\":\"2\",\"deleted-records\":\"2\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"2\",\"total-records\":\"22\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"22\",\"total-delete-files\":\"8\",\"total-position-deletes\":\"8\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"}
current-snapshot-timestamp-ms #Masked#
default-partition-spec
{\"spec-id\":1,\"fields\":[{\"name\":\"company_id\",\"transform\":\"identity\",\"source-id\":5,\"field-id\":1000},{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1001}]}
format-version 2
hive.compactor.worker.pool iceberg
#### A masked pattern was here ####
- numFiles 17
- numRows 17
+ numFiles 22
+ numRows 22
parquet.compression zstd
#### A masked pattern was here ####
rawDataSize 0
serialization.format 1
- snapshot-count 20
+ snapshot-count 25
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
totalSize #Masked#
@@ -342,6 +387,11 @@ POSTHOOK: Input: default@ice_orc
{"company_id":100,"dept_id":1,"team_id":11} 2 0 1
{"company_id":100,"dept_id":1,"team_id":11} 2 0 1
{"company_id":100,"dept_id":1,"team_id":null} 1 0 1
+{"company_id":100,"dept_id":1,"team_id":null} 1 0 1
+{"company_id":100,"dept_id":1,"team_id":null} 1 0 1
+{"company_id":100,"dept_id":1,"team_id":null} 1 0 1
+{"company_id":100,"dept_id":1,"team_id":null} 1 0 1
+{"company_id":100,"dept_id":1,"team_id":null} 1 0 1
{"company_id":100,"dept_id":1,"team_id":null} 1 1 1
{"company_id":100,"dept_id":2,"team_id":21} 2 0 1
{"company_id":100,"dept_id":2,"team_id":21} 2 0 1
@@ -428,14 +478,14 @@ STAGE PLANS:
TableScan
alias: ice_orc.tag_v2
Snapshot ref: tag_v2
- Statistics: Num rows: 8 Data size: 3136 Basic stats:
COMPLETE Column stats: NONE
+ Statistics: Num rows: 13 Data size: 5096 Basic stats:
COMPLETE Column stats: NONE
Select Operator
expressions: first_name (type: string), last_name (type:
string), dept_id (type: bigint), team_id (type: bigint), company_id (type:
bigint)
outputColumnNames: _col0, _col1, _col2, _col3, _col4
- Statistics: Num rows: 8 Data size: 3136 Basic stats:
COMPLETE Column stats: NONE
+ Statistics: Num rows: 13 Data size: 5096 Basic stats:
COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 8 Data size: 3136 Basic stats:
COMPLETE Column stats: NONE
+ Statistics: Num rows: 13 Data size: 5096 Basic stats:
COMPLETE Column stats: NONE
table:
input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -471,14 +521,14 @@ STAGE PLANS:
TableScan
alias: ice_orc.tag_v3
Snapshot ref: tag_v3
- Statistics: Num rows: 11 Data size: 4312 Basic stats:
COMPLETE Column stats: NONE
+ Statistics: Num rows: 16 Data size: 6272 Basic stats:
COMPLETE Column stats: NONE
Select Operator
expressions: first_name (type: string), last_name (type:
string), dept_id (type: bigint), team_id (type: bigint), company_id (type:
bigint)
outputColumnNames: _col0, _col1, _col2, _col3, _col4
- Statistics: Num rows: 11 Data size: 4312 Basic stats:
COMPLETE Column stats: NONE
+ Statistics: Num rows: 16 Data size: 6272 Basic stats:
COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 11 Data size: 4312 Basic stats:
COMPLETE Column stats: NONE
+ Statistics: Num rows: 16 Data size: 6272 Basic stats:
COMPLETE Column stats: NONE
table:
input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -514,14 +564,14 @@ STAGE PLANS:
TableScan
alias: ice_orc.tag_v4
Snapshot ref: tag_v4
- Statistics: Num rows: 9 Data size: 3528 Basic stats:
COMPLETE Column stats: NONE
+ Statistics: Num rows: 14 Data size: 5488 Basic stats:
COMPLETE Column stats: NONE
Select Operator
expressions: first_name (type: string), last_name (type:
string), dept_id (type: bigint), team_id (type: bigint), company_id (type:
bigint)
outputColumnNames: _col0, _col1, _col2, _col3, _col4
- Statistics: Num rows: 9 Data size: 3528 Basic stats:
COMPLETE Column stats: NONE
+ Statistics: Num rows: 14 Data size: 5488 Basic stats:
COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 9 Data size: 3528 Basic stats:
COMPLETE Column stats: NONE
+ Statistics: Num rows: 14 Data size: 5488 Basic stats:
COMPLETE Column stats: NONE
table:
input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -556,14 +606,14 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: ice_orc
- Statistics: Num rows: 9 Data size: 3528 Basic stats:
COMPLETE Column stats: NONE
+ Statistics: Num rows: 14 Data size: 5488 Basic stats:
COMPLETE Column stats: NONE
Select Operator
expressions: first_name (type: string), last_name (type:
string), dept_id (type: bigint), team_id (type: bigint), company_id (type:
bigint)
outputColumnNames: _col0, _col1, _col2, _col3, _col4
- Statistics: Num rows: 9 Data size: 3528 Basic stats:
COMPLETE Column stats: NONE
+ Statistics: Num rows: 14 Data size: 5488 Basic stats:
COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 9 Data size: 3528 Basic stats:
COMPLETE Column stats: NONE
+ Statistics: Num rows: 14 Data size: 5488 Basic stats:
COMPLETE Column stats: NONE
table:
input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -647,17 +697,17 @@ STAGE PLANS:
alias: ice_orc.tag_v2
filterExpr: company_id is not null (type: boolean)
Snapshot ref: tag_v2
- Statistics: Num rows: 8 Data size: 3136 Basic stats:
COMPLETE Column stats: NONE
+ Statistics: Num rows: 13 Data size: 5096 Basic stats:
COMPLETE Column stats: NONE
Filter Operator
predicate: company_id is not null (type: boolean)
- Statistics: Num rows: 8 Data size: 3136 Basic stats:
COMPLETE Column stats: NONE
+ Statistics: Num rows: 13 Data size: 5096 Basic stats:
COMPLETE Column stats: NONE
Select Operator
expressions: first_name (type: string), last_name (type:
string), dept_id (type: bigint), team_id (type: bigint), company_id (type:
bigint)
outputColumnNames: _col0, _col1, _col2, _col3, _col4
- Statistics: Num rows: 8 Data size: 3136 Basic stats:
COMPLETE Column stats: NONE
+ Statistics: Num rows: 13 Data size: 5096 Basic stats:
COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 8 Data size: 3136 Basic stats:
COMPLETE Column stats: NONE
+ Statistics: Num rows: 13 Data size: 5096 Basic stats:
COMPLETE Column stats: NONE
table:
input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -694,17 +744,17 @@ STAGE PLANS:
alias: ice_orc.tag_v3
filterExpr: company_id is not null (type: boolean)
Snapshot ref: tag_v3
- Statistics: Num rows: 11 Data size: 4312 Basic stats:
COMPLETE Column stats: NONE
+ Statistics: Num rows: 16 Data size: 6272 Basic stats:
COMPLETE Column stats: NONE
Filter Operator
predicate: company_id is not null (type: boolean)
- Statistics: Num rows: 11 Data size: 4312 Basic stats:
COMPLETE Column stats: NONE
+ Statistics: Num rows: 16 Data size: 6272 Basic stats:
COMPLETE Column stats: NONE
Select Operator
expressions: first_name (type: string), last_name (type:
string), dept_id (type: bigint), team_id (type: bigint), company_id (type:
bigint)
outputColumnNames: _col0, _col1, _col2, _col3, _col4
- Statistics: Num rows: 11 Data size: 4312 Basic stats:
COMPLETE Column stats: NONE
+ Statistics: Num rows: 16 Data size: 6272 Basic stats:
COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 11 Data size: 4312 Basic stats:
COMPLETE Column stats: NONE
+ Statistics: Num rows: 16 Data size: 6272 Basic stats:
COMPLETE Column stats: NONE
table:
input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -741,17 +791,17 @@ STAGE PLANS:
alias: ice_orc.tag_v4
filterExpr: company_id is not null (type: boolean)
Snapshot ref: tag_v4
- Statistics: Num rows: 9 Data size: 3528 Basic stats:
COMPLETE Column stats: NONE
+ Statistics: Num rows: 14 Data size: 5488 Basic stats:
COMPLETE Column stats: NONE
Filter Operator
predicate: company_id is not null (type: boolean)
- Statistics: Num rows: 9 Data size: 3528 Basic stats:
COMPLETE Column stats: NONE
+ Statistics: Num rows: 14 Data size: 5488 Basic stats:
COMPLETE Column stats: NONE
Select Operator
expressions: first_name (type: string), last_name (type:
string), dept_id (type: bigint), team_id (type: bigint), company_id (type:
bigint)
outputColumnNames: _col0, _col1, _col2, _col3, _col4
- Statistics: Num rows: 9 Data size: 3528 Basic stats:
COMPLETE Column stats: NONE
+ Statistics: Num rows: 14 Data size: 5488 Basic stats:
COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 9 Data size: 3528 Basic stats:
COMPLETE Column stats: NONE
+ Statistics: Num rows: 14 Data size: 5488 Basic stats:
COMPLETE Column stats: NONE
table:
input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -787,17 +837,17 @@ STAGE PLANS:
TableScan
alias: ice_orc
filterExpr: company_id is not null (type: boolean)
- Statistics: Num rows: 9 Data size: 3528 Basic stats:
COMPLETE Column stats: NONE
+ Statistics: Num rows: 14 Data size: 5488 Basic stats:
COMPLETE Column stats: NONE
Filter Operator
predicate: company_id is not null (type: boolean)
- Statistics: Num rows: 9 Data size: 3528 Basic stats:
COMPLETE Column stats: NONE
+ Statistics: Num rows: 14 Data size: 5488 Basic stats:
COMPLETE Column stats: NONE
Select Operator
expressions: first_name (type: string), last_name (type:
string), dept_id (type: bigint), team_id (type: bigint), company_id (type:
bigint)
outputColumnNames: _col0, _col1, _col2, _col3, _col4
- Statistics: Num rows: 9 Data size: 3528 Basic stats:
COMPLETE Column stats: NONE
+ Statistics: Num rows: 14 Data size: 5488 Basic stats:
COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 9 Data size: 3528 Basic stats:
COMPLETE Column stats: NONE
+ Statistics: Num rows: 14 Data size: 5488 Basic stats:
COMPLETE Column stats: NONE
table:
input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -853,6 +903,11 @@ NULL NULL NULL NULL NULL
fn2 ln2a 1 10 100
fn3 ln3a 1 11 100
fn4 ln4a 1 11 100
+fn41 ln41 1 11 100
+fn42 ln42 1 11 100
+fn43 ln43 1 11 100
+fn44 ln44 1 11 100
+fn45 ln45 1 11 100
fn5 ln5a 2 20 100
fn6 ln6a 2 20 100
fn7 ln7a 2 21 100
@@ -890,20 +945,20 @@ Table Parameters:
compactor.threshold.target.size 1500
current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]}
current-snapshot-id #Masked#
- current-snapshot-summary
{\"manifests-created\":\"6\",\"manifests-kept\":\"2\",\"manifests-replaced\":\"5\",\"added-data-files\":\"2\",\"deleted-data-files\":\"10\",\"removed-position-delete-files\":\"5\",\"removed-delete-files\":\"5\",\"added-records\":\"5\",\"deleted-records\":\"10\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"removed-position-deletes\":\"5\",\"changed-partition-count\":\"8\",\"total-records\":\"9\",\"total-files-size\":\"#Masked#\",\"total
[...]
+ current-snapshot-summary
{\"manifests-created\":\"6\",\"manifests-kept\":\"3\",\"manifests-replaced\":\"5\",\"added-data-files\":\"2\",\"deleted-data-files\":\"10\",\"removed-position-delete-files\":\"5\",\"removed-delete-files\":\"5\",\"added-records\":\"5\",\"deleted-records\":\"10\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"removed-position-deletes\":\"5\",\"changed-partition-count\":\"8\",\"total-records\":\"14\",\"total-files-size\":\"#Masked#\",\"tota
[...]
current-snapshot-timestamp-ms #Masked#
default-partition-spec
{\"spec-id\":1,\"fields\":[{\"name\":\"company_id\",\"transform\":\"identity\",\"source-id\":5,\"field-id\":1000},{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1001}]}
format-version 2
hive.compactor.worker.pool iceberg
#### A masked pattern was here ####
- numFiles 4
+ numFiles 5
numPartitions 3
- numRows 9
+ numRows 14
parquet.compression zstd
#### A masked pattern was here ####
rawDataSize 0
serialization.format 1
- snapshot-count 23
+ snapshot-count 28
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
totalSize #Masked#
@@ -943,6 +998,7 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@ice_orc
#### A masked pattern was here ####
{"company_id":100,"dept_id":1,"team_id":null} 1 0 3
+{"company_id":100,"dept_id":1,"team_id":null} 1 0 5
{"company_id":100,"dept_id":2,"team_id":null} 1 0 3
{"company_id":null,"dept_id":null,"team_id":null} 1 0 1
{"company_id":null,"dept_id":null,"team_id":null} 1 0 2
@@ -967,14 +1023,14 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: ice_orc
- Statistics: Num rows: 9 Data size: 3528 Basic stats:
COMPLETE Column stats: NONE
+ Statistics: Num rows: 14 Data size: 5488 Basic stats:
COMPLETE Column stats: NONE
Select Operator
expressions: first_name (type: string), last_name (type:
string), dept_id (type: bigint), team_id (type: bigint), company_id (type:
bigint)
outputColumnNames: _col0, _col1, _col2, _col3, _col4
- Statistics: Num rows: 9 Data size: 3528 Basic stats:
COMPLETE Column stats: NONE
+ Statistics: Num rows: 14 Data size: 5488 Basic stats:
COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 9 Data size: 3528 Basic stats:
COMPLETE Column stats: NONE
+ Statistics: Num rows: 14 Data size: 5488 Basic stats:
COMPLETE Column stats: NONE
table:
input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -1010,14 +1066,14 @@ STAGE PLANS:
TableScan
alias: ice_orc
filterExpr: company_id is not null (type: boolean)
- Statistics: Num rows: 6 Data size: 2352 Basic stats:
COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 11 Data size: 4312 Basic stats:
COMPLETE Column stats: PARTIAL
Select Operator
expressions: first_name (type: string), last_name (type:
string), dept_id (type: bigint), team_id (type: bigint), company_id (type:
bigint)
outputColumnNames: _col0, _col1, _col2, _col3, _col4
- Statistics: Num rows: 6 Data size: 2352 Basic stats:
COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 11 Data size: 4312 Basic stats:
COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
- Statistics: Num rows: 6 Data size: 2352 Basic stats:
COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 11 Data size: 4312 Basic stats:
COMPLETE Column stats: PARTIAL
table:
input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out
index 9a0f3545ca2..f1d4557954d 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out
@@ -62,12 +62,36 @@ POSTHOOK: query: insert into ice_orc VALUES ('fn4','ln4', 1)
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: ice_comp@ice_orc
-PREHOOK: query: delete from ice_orc where last_name in ('ln3', 'ln4')
+PREHOOK: query: insert into ice_orc VALUES ('fn5','ln5', 1)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: ice_comp@ice_orc
+POSTHOOK: query: insert into ice_orc VALUES ('fn5','ln5', 1)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: ice_comp@ice_orc
+PREHOOK: query: insert into ice_orc VALUES ('fn6','ln6', 1)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: ice_comp@ice_orc
+POSTHOOK: query: insert into ice_orc VALUES ('fn6','ln6', 1)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: ice_comp@ice_orc
+PREHOOK: query: insert into ice_orc VALUES ('fn7','ln7', 1)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: ice_comp@ice_orc
+POSTHOOK: query: insert into ice_orc VALUES ('fn7','ln7', 1)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: ice_comp@ice_orc
+PREHOOK: query: delete from ice_orc where last_name in ('ln6', 'ln7')
PREHOOK: type: QUERY
PREHOOK: Input: ice_comp@ice_orc
#### A masked pattern was here ####
PREHOOK: Output: ice_comp@ice_orc
-POSTHOOK: query: delete from ice_orc where last_name in ('ln3', 'ln4')
+POSTHOOK: query: delete from ice_orc where last_name in ('ln6', 'ln7')
POSTHOOK: type: QUERY
POSTHOOK: Input: ice_comp@ice_orc
#### A masked pattern was here ####
@@ -79,44 +103,68 @@ POSTHOOK: query: alter table ice_orc set partition
spec(dept_id)
POSTHOOK: type: ALTERTABLE_SETPARTSPEC
POSTHOOK: Input: ice_comp@ice_orc
POSTHOOK: Output: ice_comp@ice_orc
-PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn5','ln5')
+PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn8','ln8')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: ice_comp@ice_orc@dept_id=2
-POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn5','ln5')
+POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn8','ln8')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: ice_comp@ice_orc@dept_id=2
-PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn6','ln6')
+PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn9','ln9')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: ice_comp@ice_orc@dept_id=2
-POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn6','ln6')
+POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn9','ln9')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: ice_comp@ice_orc@dept_id=2
-PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn7','ln7')
+PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn10','ln10')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: ice_comp@ice_orc@dept_id=2
-POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn7','ln7')
+POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES
('fn10','ln10')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: ice_comp@ice_orc@dept_id=2
-PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn8','ln8')
+PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn11','ln11')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: ice_comp@ice_orc@dept_id=2
-POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn8','ln8')
+POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES
('fn11','ln11')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: ice_comp@ice_orc@dept_id=2
+PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn12','ln12')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: ice_comp@ice_orc@dept_id=2
+POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES
('fn12','ln12')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: ice_comp@ice_orc@dept_id=2
+PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn13','ln13')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: ice_comp@ice_orc@dept_id=2
+POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES
('fn13','ln13')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: ice_comp@ice_orc@dept_id=2
+PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn14','ln14')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: ice_comp@ice_orc@dept_id=2
+POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES
('fn14','ln14')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: ice_comp@ice_orc@dept_id=2
-PREHOOK: query: delete from ice_orc where last_name in ('ln7', 'ln8')
+PREHOOK: query: delete from ice_orc where last_name in ('ln13', 'ln14')
PREHOOK: type: QUERY
PREHOOK: Input: ice_comp@ice_orc
#### A masked pattern was here ####
PREHOOK: Output: ice_comp@ice_orc
-POSTHOOK: query: delete from ice_orc where last_name in ('ln7', 'ln8')
+POSTHOOK: query: delete from ice_orc where last_name in ('ln13', 'ln14')
POSTHOOK: type: QUERY
POSTHOOK: Input: ice_comp@ice_orc
#### A masked pattern was here ####
@@ -130,9 +178,15 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: ice_comp@ice_orc
#### A masked pattern was here ####
fn1 ln1 1
+fn10 ln10 2
+fn11 ln11 2
+fn12 ln12 2
fn2 ln2 1
-fn5 ln5 2
-fn6 ln6 2
+fn3 ln3 1
+fn4 ln4 1
+fn5 ln5 1
+fn8 ln8 2
+fn9 ln9 2
PREHOOK: query: describe formatted ice_orc
PREHOOK: type: DESCTABLE
PREHOOK: Input: ice_comp@ice_orc
@@ -164,18 +218,18 @@ Table Parameters:
compactor.threshold.target.size 1500
current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"}]}
current-snapshot-id #Masked#
- current-snapshot-summary
{\"manifests-created\":\"2\",\"manifests-kept\":\"3\",\"manifests-replaced\":\"2\",\"deleted-data-files\":\"2\",\"deleted-records\":\"2\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"4\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"4\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"}
+ current-snapshot-summary
{\"manifests-created\":\"2\",\"manifests-kept\":\"6\",\"manifests-replaced\":\"2\",\"deleted-data-files\":\"2\",\"deleted-records\":\"2\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"10\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"10\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"}
current-snapshot-timestamp-ms #Masked#
default-partition-spec
{\"spec-id\":1,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000}]}
format-version 2
#### A masked pattern was here ####
- numFiles 4
- numRows 4
+ numFiles 10
+ numRows 10
parquet.compression zstd
#### A masked pattern was here ####
rawDataSize 0
serialization.format 1
- snapshot-count 10
+ snapshot-count 16
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
totalSize #Masked#
@@ -215,9 +269,15 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: ice_comp@ice_orc
#### A masked pattern was here ####
fn1 ln1 1
+fn10 ln10 2
+fn11 ln11 2
+fn12 ln12 2
fn2 ln2 1
-fn5 ln5 2
-fn6 ln6 2
+fn3 ln3 1
+fn4 ln4 1
+fn5 ln5 1
+fn8 ln8 2
+fn9 ln9 2
PREHOOK: query: describe formatted ice_orc
PREHOOK: type: DESCTABLE
PREHOOK: Input: ice_comp@ice_orc
@@ -249,19 +309,19 @@ Table Parameters:
compactor.threshold.target.size 1500
current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"}]}
current-snapshot-id #Masked#
- current-snapshot-summary
{\"manifests-created\":\"2\",\"manifests-kept\":\"1\",\"manifests-replaced\":\"1\",\"added-data-files\":\"1\",\"deleted-data-files\":\"2\",\"added-records\":\"2\",\"deleted-records\":\"2\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"2\",\"total-records\":\"4\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"2\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\"
[...]
+ current-snapshot-summary
{\"manifests-created\":\"2\",\"manifests-kept\":\"1\",\"manifests-replaced\":\"1\",\"added-data-files\":\"1\",\"deleted-data-files\":\"5\",\"added-records\":\"5\",\"deleted-records\":\"5\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"2\",\"total-records\":\"10\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"2\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\
[...]
current-snapshot-timestamp-ms #Masked#
default-partition-spec
{\"spec-id\":1,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000}]}
format-version 2
#### A masked pattern was here ####
numFiles 2
numPartitions 2
- numRows 4
+ numRows 10
parquet.compression zstd
#### A masked pattern was here ####
rawDataSize 0
serialization.format 1
- snapshot-count 12
+ snapshot-count 18
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
totalSize #Masked#
@@ -293,19 +353,59 @@ POSTHOOK: query: alter table ice_orc set partition spec()
POSTHOOK: type: ALTERTABLE_SETPARTSPEC
POSTHOOK: Input: ice_comp@ice_orc
POSTHOOK: Output: ice_comp@ice_orc
-PREHOOK: query: insert into ice_orc VALUES ('fn9','ln9', 3)
+PREHOOK: query: insert into ice_orc VALUES ('fn15','ln15', 3)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: ice_comp@ice_orc
+POSTHOOK: query: insert into ice_orc VALUES ('fn15','ln15', 3)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: ice_comp@ice_orc
+PREHOOK: query: insert into ice_orc VALUES ('fn16','ln16', 3)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: ice_comp@ice_orc
+POSTHOOK: query: insert into ice_orc VALUES ('fn16','ln16', 3)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: ice_comp@ice_orc
+PREHOOK: query: insert into ice_orc VALUES ('fn17','ln17', 3)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: ice_comp@ice_orc
+POSTHOOK: query: insert into ice_orc VALUES ('fn17','ln17', 3)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: ice_comp@ice_orc
+PREHOOK: query: insert into ice_orc VALUES ('fn18','ln18', 3)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: ice_comp@ice_orc
+POSTHOOK: query: insert into ice_orc VALUES ('fn18','ln18', 3)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: ice_comp@ice_orc
+PREHOOK: query: insert into ice_orc VALUES ('fn19','ln19', 3)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: ice_comp@ice_orc
+POSTHOOK: query: insert into ice_orc VALUES ('fn19','ln19', 3)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: ice_comp@ice_orc
+PREHOOK: query: insert into ice_orc VALUES ('fn20','ln20', 3)
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: ice_comp@ice_orc
-POSTHOOK: query: insert into ice_orc VALUES ('fn9','ln9', 3)
+POSTHOOK: query: insert into ice_orc VALUES ('fn20','ln20', 3)
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: ice_comp@ice_orc
-PREHOOK: query: insert into ice_orc VALUES ('fn10','ln10', 3)
+PREHOOK: query: insert into ice_orc VALUES ('fn21','ln21', 3)
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: ice_comp@ice_orc
-POSTHOOK: query: insert into ice_orc VALUES ('fn10','ln10', 3)
+POSTHOOK: query: insert into ice_orc VALUES ('fn21','ln21', 3)
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: ice_comp@ice_orc
@@ -318,11 +418,22 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: ice_comp@ice_orc
#### A masked pattern was here ####
fn1 ln1 1
-fn10 ln10 3
+fn10 ln10 2
+fn11 ln11 2
+fn12 ln12 2
+fn15 ln15 3
+fn16 ln16 3
+fn17 ln17 3
+fn18 ln18 3
+fn19 ln19 3
fn2 ln2 1
-fn5 ln5 2
-fn6 ln6 2
-fn9 ln9 3
+fn20 ln20 3
+fn21 ln21 3
+fn3 ln3 1
+fn4 ln4 1
+fn5 ln5 1
+fn8 ln8 2
+fn9 ln9 2
PREHOOK: query: describe formatted ice_orc
PREHOOK: type: DESCTABLE
PREHOOK: Input: ice_comp@ice_orc
@@ -347,17 +458,17 @@ Table Parameters:
compactor.threshold.target.size 1500
current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"}]}
current-snapshot-id #Masked#
- current-snapshot-summary
{\"manifests-created\":\"1\",\"manifests-kept\":\"2\",\"manifests-replaced\":\"0\",\"added-data-files\":\"1\",\"added-records\":\"1\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"6\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"4\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"}
+ current-snapshot-summary
{\"manifests-created\":\"1\",\"manifests-kept\":\"7\",\"manifests-replaced\":\"0\",\"added-data-files\":\"1\",\"added-records\":\"1\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"17\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"9\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"}
current-snapshot-timestamp-ms #Masked#
format-version 2
#### A masked pattern was here ####
- numFiles 4
- numRows 6
+ numFiles 9
+ numRows 17
parquet.compression zstd
#### A masked pattern was here ####
rawDataSize 0
serialization.format 1
- snapshot-count 14
+ snapshot-count 25
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
totalSize #Masked#
@@ -392,11 +503,22 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: ice_comp@ice_orc
#### A masked pattern was here ####
fn1 ln1 1
-fn10 ln10 3
+fn10 ln10 2
+fn11 ln11 2
+fn12 ln12 2
+fn15 ln15 3
+fn16 ln16 3
+fn17 ln17 3
+fn18 ln18 3
+fn19 ln19 3
fn2 ln2 1
-fn5 ln5 2
-fn6 ln6 2
-fn9 ln9 3
+fn20 ln20 3
+fn21 ln21 3
+fn3 ln3 1
+fn4 ln4 1
+fn5 ln5 1
+fn8 ln8 2
+fn9 ln9 2
PREHOOK: query: describe formatted ice_orc
PREHOOK: type: DESCTABLE
PREHOOK: Input: ice_comp@ice_orc
@@ -421,17 +543,17 @@ Table Parameters:
compactor.threshold.target.size 1500
current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"}]}
current-snapshot-id #Masked#
- current-snapshot-summary
{\"manifests-created\":\"4\",\"manifests-kept\":\"0\",\"manifests-replaced\":\"3\",\"added-data-files\":\"1\",\"deleted-data-files\":\"4\",\"added-records\":\"6\",\"deleted-records\":\"6\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"3\",\"total-records\":\"6\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\"
[...]
+ current-snapshot-summary
{\"manifests-created\":\"9\",\"manifests-kept\":\"0\",\"manifests-replaced\":\"8\",\"added-data-files\":\"1\",\"deleted-data-files\":\"9\",\"added-records\":\"17\",\"deleted-records\":\"17\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"3\",\"total-records\":\"17\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-delete
[...]
current-snapshot-timestamp-ms #Masked#
format-version 2
#### A masked pattern was here ####
numFiles 1
- numRows 6
+ numRows 17
parquet.compression zstd
#### A masked pattern was here ####
rawDataSize 0
serialization.format 1
- snapshot-count 15
+ snapshot-count 26
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
totalSize #Masked#
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out
index b59449ea645..c4acdcaade3 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out
@@ -69,6 +69,46 @@ POSTHOOK: query: insert into ice_orc VALUES
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc VALUES ('fn81','ln81', 2, 20, 101)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc VALUES ('fn81','ln81', 2, 20, 101)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc VALUES ('fn82','ln82', 2, 20, 101)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc VALUES ('fn82','ln82', 2, 20, 101)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc VALUES ('fn83','ln83', 2, 20, 101)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc VALUES ('fn83','ln83', 2, 20, 101)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc VALUES ('fn84','ln84', 2, 20, 101)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc VALUES ('fn84','ln84', 2, 20, 101)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc VALUES ('fn85','ln85', 2, 20, 101)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc VALUES ('fn85','ln85', 2, 20, 101)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
PREHOOK: query: insert into ice_orc VALUES
('fn9', 'ln9', 3, 20, 101),
('fn10','ln10', 3, 20, 101),
@@ -109,6 +149,11 @@ fn5 ln5 2 20 101
fn6 ln6 2 20 101
fn7 ln7 2 20 101
fn8 ln8 2 20 101
+fn81 ln81 2 20 101
+fn82 ln82 2 20 101
+fn83 ln83 2 20 101
+fn84 ln84 2 20 101
+fn85 ln85 2 20 101
PREHOOK: query: select * from ice_orc where dept_id = 3 order by first_name
PREHOOK: type: QUERY
PREHOOK: Input: default@ice_orc
@@ -154,19 +199,19 @@ Table Parameters:
compactor.threshold.target.size 1500
current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]}
current-snapshot-id #Masked#
- current-snapshot-summary
{\"manifests-created\":\"1\",\"manifests-kept\":\"3\",\"manifests-replaced\":\"0\",\"added-data-files\":\"1\",\"added-records\":\"4\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"12\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"4\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"}
+ current-snapshot-summary
{\"manifests-created\":\"1\",\"manifests-kept\":\"8\",\"manifests-replaced\":\"0\",\"added-data-files\":\"1\",\"added-records\":\"4\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"17\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"9\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"}
current-snapshot-timestamp-ms #Masked#
default-partition-spec
{\"spec-id\":1,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1001}]}
format-version 2
#### A masked pattern was here ####
- numFiles 4
+ numFiles 9
numPartitions 3
- numRows 12
+ numRows 17
parquet.compression zstd
#### A masked pattern was here ####
rawDataSize 0
serialization.format 1
- snapshot-count 4
+ snapshot-count 9
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
totalSize #Masked#
@@ -253,6 +298,11 @@ POSTHOOK: query: select * from ice_orc where dept_id = 2
POSTHOOK: type: QUERY
POSTHOOK: Input: default@ice_orc
#### A masked pattern was here ####
+fn85 ln85 2 20 101
+fn84 ln84 2 20 101
+fn83 ln83 2 20 101
+fn82 ln82 2 20 101
+fn81 ln81 2 20 101
fn8 ln8 2 20 101
fn7 ln7 2 20 101
fn6 ln6 2 20 101
@@ -302,19 +352,19 @@ Table Parameters:
compactor.threshold.target.size 1500
current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]}
current-snapshot-id #Masked#
- current-snapshot-summary
{\"manifests-created\":\"3\",\"manifests-kept\":\"2\",\"manifests-replaced\":\"2\",\"added-data-files\":\"1\",\"deleted-data-files\":\"2\",\"added-records\":\"4\",\"deleted-records\":\"4\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"12\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"3\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\
[...]
+ current-snapshot-summary
{\"manifests-created\":\"8\",\"manifests-kept\":\"2\",\"manifests-replaced\":\"7\",\"added-data-files\":\"1\",\"deleted-data-files\":\"7\",\"added-records\":\"9\",\"deleted-records\":\"9\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"17\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"3\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\
[...]
current-snapshot-timestamp-ms #Masked#
default-partition-spec
{\"spec-id\":1,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1001}]}
format-version 2
#### A masked pattern was here ####
numFiles 3
numPartitions 3
- numRows 12
+ numRows 17
parquet.compression zstd
#### A masked pattern was here ####
rawDataSize 0
serialization.format 1
- snapshot-count 5
+ snapshot-count 10
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
totalSize #Masked#
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out
index 88e5686f05a..97ec86f07a7 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out
@@ -20,50 +20,82 @@ tblproperties ('compactor.threshold.target.size'='1500')
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc values
- (1, cast('2023-07-20 00:00:00' as timestamp with local time zone),
'AAA_1'),
+PREHOOK: query: insert into ice_orc values
+ (1, cast('2023-07-20 00:00:00' as timestamp with local time zone), 'AAA_1')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc values
+ (1, cast('2023-07-20 00:00:00' as timestamp with local time zone), 'AAA_1')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc values
(2, cast('2023-07-21 00:00:00' as timestamp with local time zone), 'AAA_2')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc values
- (1, cast('2023-07-20 00:00:00' as timestamp with local time zone),
'AAA_1'),
+POSTHOOK: query: insert into ice_orc values
(2, cast('2023-07-21 00:00:00' as timestamp with local time zone), 'AAA_2')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc values
- (3, cast('2023-07-23 00:00:00' as timestamp with local time zone),
'AAA_1'),
+PREHOOK: query: insert into ice_orc values
+ (3, cast('2023-07-23 00:00:00' as timestamp with local time zone), 'AAA_1')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc values
+ (3, cast('2023-07-23 00:00:00' as timestamp with local time zone), 'AAA_1')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc values
(4, cast('2023-07-24 00:00:00' as timestamp with local time zone), 'AAA_2')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc values
- (3, cast('2023-07-23 00:00:00' as timestamp with local time zone),
'AAA_1'),
+POSTHOOK: query: insert into ice_orc values
(4, cast('2023-07-24 00:00:00' as timestamp with local time zone), 'AAA_2')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc values
- (5, cast('2023-08-04 00:00:00' as timestamp with local time zone),
'BBB_1'),
+PREHOOK: query: insert into ice_orc values
+ (5, cast('2023-08-04 00:00:00' as timestamp with local time zone), 'BBB_1')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc values
+ (5, cast('2023-08-04 00:00:00' as timestamp with local time zone), 'BBB_1')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc values
(6, cast('2023-08-05 00:00:00' as timestamp with local time zone), 'BBB_2')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc values
- (5, cast('2023-08-04 00:00:00' as timestamp with local time zone),
'BBB_1'),
+POSTHOOK: query: insert into ice_orc values
(6, cast('2023-08-05 00:00:00' as timestamp with local time zone), 'BBB_2')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc values
- (7, cast('2023-08-06 00:00:00' as timestamp with local time zone),
'BBB_1'),
+PREHOOK: query: insert into ice_orc values
+ (7, cast('2023-08-06 00:00:00' as timestamp with local time zone), 'BBB_1')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc values
+ (7, cast('2023-08-06 00:00:00' as timestamp with local time zone), 'BBB_1')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc values
(8, cast('2023-08-27 00:00:00' as timestamp with local time zone), 'BBB_2')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc values
- (7, cast('2023-08-06 00:00:00' as timestamp with local time zone),
'BBB_1'),
+POSTHOOK: query: insert into ice_orc values
(8, cast('2023-08-27 00:00:00' as timestamp with local time zone), 'BBB_2')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
@@ -75,110 +107,182 @@ POSTHOOK: query: alter table ice_orc set partition
spec(truncate(3, event_src),
POSTHOOK: type: ALTERTABLE_SETPARTSPEC
POSTHOOK: Input: default@ice_orc
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc values
- (9, cast('2024-07-20 00:00:00' as timestamp with local time zone),
'AAA_1'),
+PREHOOK: query: insert into ice_orc values
+ (9, cast('2024-07-20 00:00:00' as timestamp with local time zone), 'AAA_1')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc values
+ (9, cast('2024-07-20 00:00:00' as timestamp with local time zone), 'AAA_1')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc values
(10, cast('2024-07-21 00:00:00' as timestamp with local time zone),
'AAA_2')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc values
- (9, cast('2024-07-20 00:00:00' as timestamp with local time zone),
'AAA_1'),
+POSTHOOK: query: insert into ice_orc values
(10, cast('2024-07-21 00:00:00' as timestamp with local time zone),
'AAA_2')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc values
- (11, cast('2024-08-22 00:00:00' as timestamp with local time zone),
'AAA_1'),
+PREHOOK: query: insert into ice_orc values
+ (11, cast('2024-08-22 00:00:00' as timestamp with local time zone),
'AAA_1')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc values
+ (11, cast('2024-08-22 00:00:00' as timestamp with local time zone),
'AAA_1')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc values
(12, cast('2024-08-23 00:00:00' as timestamp with local time zone),
'AAA_2')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc values
- (11, cast('2024-08-22 00:00:00' as timestamp with local time zone),
'AAA_1'),
+POSTHOOK: query: insert into ice_orc values
(12, cast('2024-08-23 00:00:00' as timestamp with local time zone),
'AAA_2')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc values
- (13, cast('2024-08-24 00:00:00' as timestamp with local time zone),
'AAA_1'),
+PREHOOK: query: insert into ice_orc values
+ (13, cast('2024-08-24 00:00:00' as timestamp with local time zone),
'AAA_1')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc values
+ (13, cast('2024-08-24 00:00:00' as timestamp with local time zone),
'AAA_1')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc values
(14, cast('2024-08-25 00:00:00' as timestamp with local time zone),
'AAA_2')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc values
- (13, cast('2024-08-24 00:00:00' as timestamp with local time zone),
'AAA_1'),
+POSTHOOK: query: insert into ice_orc values
(14, cast('2024-08-25 00:00:00' as timestamp with local time zone),
'AAA_2')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc values
- (15, cast('2024-09-24 00:00:00' as timestamp with local time zone),
'AAA_1'),
+PREHOOK: query: insert into ice_orc values
+ (15, cast('2024-09-24 00:00:00' as timestamp with local time zone),
'AAA_1')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc values
+ (15, cast('2024-09-24 00:00:00' as timestamp with local time zone),
'AAA_1')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc values
(16, cast('2024-09-25 00:00:00' as timestamp with local time zone),
'AAA_2')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc values
- (15, cast('2024-09-24 00:00:00' as timestamp with local time zone),
'AAA_1'),
+POSTHOOK: query: insert into ice_orc values
(16, cast('2024-09-25 00:00:00' as timestamp with local time zone),
'AAA_2')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc values
- (17, cast('2024-07-01 00:00:00' as timestamp with local time zone),
'BBB_1'),
+PREHOOK: query: insert into ice_orc values
+ (17, cast('2024-07-01 00:00:00' as timestamp with local time zone),
'BBB_1')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc values
+ (17, cast('2024-07-01 00:00:00' as timestamp with local time zone),
'BBB_1')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc values
(18, cast('2024-07-02 00:00:00' as timestamp with local time zone),
'BBB_2')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc values
- (17, cast('2024-07-01 00:00:00' as timestamp with local time zone),
'BBB_1'),
+POSTHOOK: query: insert into ice_orc values
(18, cast('2024-07-02 00:00:00' as timestamp with local time zone),
'BBB_2')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc values
- (19, cast('2024-07-03 00:00:00' as timestamp with local time zone),
'BBB_1'),
+PREHOOK: query: insert into ice_orc values
+ (19, cast('2024-07-03 00:00:00' as timestamp with local time zone),
'BBB_1')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc values
+ (19, cast('2024-07-03 00:00:00' as timestamp with local time zone),
'BBB_1')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc values
(20, cast('2024-07-04 00:00:00' as timestamp with local time zone),
'BBB_2')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc values
- (19, cast('2024-07-03 00:00:00' as timestamp with local time zone),
'BBB_1'),
+POSTHOOK: query: insert into ice_orc values
(20, cast('2024-07-04 00:00:00' as timestamp with local time zone),
'BBB_2')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc values
- (21, cast('2024-08-03 00:00:00' as timestamp with local time zone),
'BBB_1'),
+PREHOOK: query: insert into ice_orc values
+ (21, cast('2024-08-03 00:00:00' as timestamp with local time zone),
'BBB_1')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc values
+ (21, cast('2024-08-03 00:00:00' as timestamp with local time zone),
'BBB_1')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc values
(22, cast('2024-08-04 00:00:00' as timestamp with local time zone),
'BBB_2')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc values
- (21, cast('2024-08-03 00:00:00' as timestamp with local time zone),
'BBB_1'),
+POSTHOOK: query: insert into ice_orc values
(22, cast('2024-08-04 00:00:00' as timestamp with local time zone),
'BBB_2')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc values
- (23, cast('2024-08-05 00:00:00' as timestamp with local time zone),
'BBB_1'),
+PREHOOK: query: insert into ice_orc values
+ (23, cast('2024-08-05 00:00:00' as timestamp with local time zone),
'BBB_1')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc values
+ (23, cast('2024-08-05 00:00:00' as timestamp with local time zone),
'BBB_1')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc values
(24, cast('2024-08-06 00:00:00' as timestamp with local time zone),
'BBB_2')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc values
- (23, cast('2024-08-05 00:00:00' as timestamp with local time zone),
'BBB_1'),
+POSTHOOK: query: insert into ice_orc values
(24, cast('2024-08-06 00:00:00' as timestamp with local time zone),
'BBB_2')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_orc
PREHOOK: query: insert into ice_orc values
- (25, cast('2024-09-05 00:00:00' as timestamp with local time zone),
'BBB_1'),
+ (25, cast('2024-09-05 00:00:00' as timestamp with local time zone),
'BBB_1')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc values
+ (25, cast('2024-09-05 00:00:00' as timestamp with local time zone),
'BBB_1')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc values
(26, cast('2024-09-06 00:00:00' as timestamp with local time zone),
'BBB_2')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_orc
POSTHOOK: query: insert into ice_orc values
- (25, cast('2024-09-05 00:00:00' as timestamp with local time zone),
'BBB_1'),
(26, cast('2024-09-06 00:00:00' as timestamp with local time zone),
'BBB_2')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
@@ -249,19 +353,19 @@ Table Parameters:
compactor.threshold.target.size 1500
current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"event_id\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"event_time\",\"required\":false,\"type\":\"timestamptz\"},{\"id\":3,\"name\":\"event_src\",\"required\":false,\"type\":\"string\"}]}
current-snapshot-id #Masked#
- current-snapshot-summary
{\"manifests-created\":\"1\",\"manifests-kept\":\"9\",\"manifests-replaced\":\"0\",\"added-data-files\":\"1\",\"added-records\":\"2\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"26\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"13\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"}
+ current-snapshot-summary
{\"manifests-created\":\"1\",\"manifests-kept\":\"18\",\"manifests-replaced\":\"0\",\"added-data-files\":\"1\",\"added-records\":\"1\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"26\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"26\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"}
current-snapshot-timestamp-ms #Masked#
default-partition-spec
{\"spec-id\":1,\"fields\":[{\"name\":\"event_src_trunc\",\"transform\":\"truncate[3]\",\"source-id\":3,\"field-id\":1000},{\"name\":\"event_time_month\",\"transform\":\"month\",\"source-id\":2,\"field-id\":1001}]}
format-version 2
#### A masked pattern was here ####
- numFiles 13
+ numFiles 26
numPartitions 8
numRows 26
parquet.compression zstd
#### A masked pattern was here ####
rawDataSize 0
serialization.format 1
- snapshot-count 13
+ snapshot-count 26
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
totalSize #Masked#
@@ -357,19 +461,19 @@ Table Parameters:
compactor.threshold.target.size 1500
current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"event_id\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"event_time\",\"required\":false,\"type\":\"timestamptz\"},{\"id\":3,\"name\":\"event_src\",\"required\":false,\"type\":\"string\"}]}
current-snapshot-id #Masked#
- current-snapshot-summary
{\"manifests-created\":\"3\",\"manifests-kept\":\"6\",\"manifests-replaced\":\"2\",\"added-data-files\":\"1\",\"deleted-data-files\":\"2\",\"added-records\":\"4\",\"deleted-records\":\"4\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"26\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"10\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes
[...]
+ current-snapshot-summary
{\"manifests-created\":\"5\",\"manifests-kept\":\"9\",\"manifests-replaced\":\"4\",\"added-data-files\":\"1\",\"deleted-data-files\":\"4\",\"added-records\":\"4\",\"deleted-records\":\"4\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"26\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"17\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes
[...]
current-snapshot-timestamp-ms #Masked#
default-partition-spec
{\"spec-id\":1,\"fields\":[{\"name\":\"event_src_trunc\",\"transform\":\"truncate[3]\",\"source-id\":3,\"field-id\":1000},{\"name\":\"event_time_month\",\"transform\":\"month\",\"source-id\":2,\"field-id\":1001}]}
format-version 2
#### A masked pattern was here ####
- numFiles 10
+ numFiles 17
numPartitions 8
numRows 26
parquet.compression zstd
#### A masked pattern was here ####
rawDataSize 0
serialization.format 1
- snapshot-count 16
+ snapshot-count 29
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
totalSize #Masked#
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_query_metadata.q.out
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_query_metadata.q.out
index e12af3d3b66..6da52474f3e 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_query_metadata.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_query_metadata.q.out
@@ -10,33 +10,59 @@ TBLPROPERTIES ('external.table.purge'='true',
'format-version'='2', 'compactor.t
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@x
-PREHOOK: query: insert into x values
-('amy', 35, 123412344),
-('adxfvy', 36, 123412534),
-('amsdfyy', 37, 123417234),
-('asafmy', 38, 123412534)
+PREHOOK: query: insert into x values ('amy', 35, 123412344)
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@x
-POSTHOOK: query: insert into x values
-('amy', 35, 123412344),
-('adxfvy', 36, 123412534),
-('amsdfyy', 37, 123417234),
-('asafmy', 38, 123412534)
+POSTHOOK: query: insert into x values ('amy', 35, 123412344)
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@x
-PREHOOK: query: insert into x values
-('amerqwy', 39, 123441234),
-('amyxzcv', 40, 123341234),
-('erweramy', 45, 122341234)
+PREHOOK: query: insert into x values ('adxfvy', 36, 123412534)
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@x
-POSTHOOK: query: insert into x values
-('amerqwy', 39, 123441234),
-('amyxzcv', 40, 123341234),
-('erweramy', 45, 122341234)
+POSTHOOK: query: insert into x values ('adxfvy', 36, 123412534)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@x
+PREHOOK: query: insert into x values ('amsdfyy', 37, 123417234)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@x
+POSTHOOK: query: insert into x values ('amsdfyy', 37, 123417234)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@x
+PREHOOK: query: insert into x values ('asafmy', 38, 123412534)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@x
+POSTHOOK: query: insert into x values ('asafmy', 38, 123412534)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@x
+PREHOOK: query: insert into x values ('amerqwy', 39, 123441234)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@x
+POSTHOOK: query: insert into x values ('amerqwy', 39, 123441234)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@x
+PREHOOK: query: insert into x values ('amyxzcv', 40, 123341234)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@x
+POSTHOOK: query: insert into x values ('amyxzcv', 40, 123341234)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@x
+PREHOOK: query: insert into x values ('erweramy', 45, 122341234)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@x
+POSTHOOK: query: insert into x values ('erweramy', 45, 122341234)
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@x
@@ -56,7 +82,7 @@ POSTHOOK: query: select count(*) from default.x.data_files
POSTHOOK: type: QUERY
POSTHOOK: Input: default@x
#### A masked pattern was here ####
-2
+7
PREHOOK: query: alter table x compact 'major' and wait
PREHOOK: type: ALTERTABLE_COMPACT
PREHOOK: Input: default@x
@@ -95,7 +121,7 @@ Table Parameters:
compactor.threshold.target.size 1500
current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"age\",\"required\":false,\"type\":\"int\"},{\"id\":3,\"name\":\"num_clicks\",\"required\":false,\"type\":\"long\"}]}
current-snapshot-id #Masked#
- current-snapshot-summary
{\"manifests-created\":\"3\",\"manifests-kept\":\"0\",\"manifests-replaced\":\"2\",\"added-data-files\":\"1\",\"deleted-data-files\":\"2\",\"added-records\":\"7\",\"deleted-records\":\"7\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"7\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\"
[...]
+ current-snapshot-summary
{\"manifests-created\":\"8\",\"manifests-kept\":\"0\",\"manifests-replaced\":\"7\",\"added-data-files\":\"1\",\"deleted-data-files\":\"7\",\"added-records\":\"7\",\"deleted-records\":\"7\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"7\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\"
[...]
current-snapshot-timestamp-ms #Masked#
external.table.purge true
format-version 2
@@ -106,7 +132,7 @@ Table Parameters:
#### A masked pattern was here ####
rawDataSize 0
serialization.format 1
- snapshot-count 3
+ snapshot-count 8
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
totalSize #Masked#
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out
index 0de0774396b..6293d8ae21b 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out
@@ -46,6 +46,22 @@ POSTHOOK: query: insert into ice_orc VALUES ('fn4','ln4',
1), ('fn3','ln3', 1)
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: ice_comp@ice_orc
+PREHOOK: query: insert into ice_orc VALUES ('fn15','ln15', 1)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: ice_comp@ice_orc
+POSTHOOK: query: insert into ice_orc VALUES ('fn15','ln15', 1)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: ice_comp@ice_orc
+PREHOOK: query: insert into ice_orc VALUES ('fn16','ln16', 1)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: ice_comp@ice_orc
+POSTHOOK: query: insert into ice_orc VALUES ('fn16','ln16', 1)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: ice_comp@ice_orc
PREHOOK: query: delete from ice_orc where last_name in ('ln4')
PREHOOK: type: QUERY
PREHOOK: Input: ice_comp@ice_orc
@@ -77,6 +93,22 @@ POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2)
VALUES ('fn8','ln8'),
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: ice_comp@ice_orc@dept_id=2
+PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn17','ln17')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: ice_comp@ice_orc@dept_id=2
+POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES
('fn17','ln17')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: ice_comp@ice_orc@dept_id=2
+PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn18','ln18')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: ice_comp@ice_orc@dept_id=2
+POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES
('fn18','ln18')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: ice_comp@ice_orc@dept_id=2
PREHOOK: query: delete from ice_orc where last_name in ('ln8')
PREHOOK: type: QUERY
PREHOOK: Input: ice_comp@ice_orc
@@ -115,19 +147,19 @@ Table Parameters:
bucketing_version 2
current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"}]}
current-snapshot-id #Masked#
- current-snapshot-summary
{\"manifests-created\":\"1\",\"manifests-kept\":\"4\",\"manifests-replaced\":\"0\",\"added-position-delete-files\":\"1\",\"added-delete-files\":\"1\",\"added-files-size\":\"#Masked#\",\"added-position-deletes\":\"1\",\"changed-partition-count\":\"1\",\"total-records\":\"8\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"4\",\"total-delete-files\":\"2\",\"total-position-deletes\":\"2\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"}
+ current-snapshot-summary
{\"manifests-created\":\"1\",\"manifests-kept\":\"6\",\"manifests-replaced\":\"0\",\"added-position-delete-files\":\"1\",\"added-delete-files\":\"1\",\"added-files-size\":\"#Masked#\",\"added-position-deletes\":\"1\",\"changed-partition-count\":\"1\",\"total-records\":\"12\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"8\",\"total-delete-files\":\"2\",\"total-position-deletes\":\"2\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"}
current-snapshot-timestamp-ms #Masked#
default-partition-spec
{\"spec-id\":1,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000}]}
format-version 2
#### A masked pattern was here ####
- numFiles 4
+ numFiles 8
numPartitions 1
- numRows 6
+ numRows 10
parquet.compression zstd
#### A masked pattern was here ####
rawDataSize 0
serialization.format 1
- snapshot-count 6
+ snapshot-count 10
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
totalSize #Masked#
@@ -198,19 +230,19 @@ Table Parameters:
compactor.threshold.target.size 3200
current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"}]}
current-snapshot-id #Masked#
- current-snapshot-summary
{\"manifests-created\":\"1\",\"manifests-kept\":\"4\",\"manifests-replaced\":\"0\",\"added-position-delete-files\":\"1\",\"added-delete-files\":\"1\",\"added-files-size\":\"#Masked#\",\"added-position-deletes\":\"1\",\"changed-partition-count\":\"1\",\"total-records\":\"8\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"4\",\"total-delete-files\":\"2\",\"total-position-deletes\":\"2\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"}
+ current-snapshot-summary
{\"manifests-created\":\"1\",\"manifests-kept\":\"6\",\"manifests-replaced\":\"0\",\"added-position-delete-files\":\"1\",\"added-delete-files\":\"1\",\"added-files-size\":\"#Masked#\",\"added-position-deletes\":\"1\",\"changed-partition-count\":\"1\",\"total-records\":\"12\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"8\",\"total-delete-files\":\"2\",\"total-position-deletes\":\"2\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"}
current-snapshot-timestamp-ms #Masked#
default-partition-spec
{\"spec-id\":1,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000}]}
format-version 2
#### A masked pattern was here ####
- numFiles 4
+ numFiles 8
numPartitions 1
- numRows 6
+ numRows 10
parquet.compression zstd
#### A masked pattern was here ####
rawDataSize 0
serialization.format 1
- snapshot-count 6
+ snapshot-count 10
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
totalSize #Masked#
@@ -260,6 +292,8 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: ice_comp@ice_orc
#### A masked pattern was here ####
fn1 ln1 1
+fn15 ln15 1
+fn16 ln16 1
fn2 ln2 1
fn3 ln3 1
PREHOOK: query: select * from ice_orc where dept_id=2
@@ -270,6 +304,8 @@ POSTHOOK: query: select * from ice_orc where dept_id=2
POSTHOOK: type: QUERY
POSTHOOK: Input: ice_comp@ice_orc
#### A masked pattern was here ####
+fn17 ln17 2
+fn18 ln18 2
fn5 ln5 2
fn6 ln6 2
fn7 ln7 2
@@ -305,19 +341,19 @@ Table Parameters:
compactor.threshold.target.size 4000
current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"}]}
current-snapshot-id #Masked#
- current-snapshot-summary
{\"manifests-created\":\"4\",\"manifests-kept\":\"1\",\"manifests-replaced\":\"3\",\"added-data-files\":\"1\",\"deleted-data-files\":\"2\",\"removed-position-delete-files\":\"2\",\"removed-delete-files\":\"2\",\"added-records\":\"3\",\"deleted-records\":\"4\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"removed-position-deletes\":\"2\",\"changed-partition-count\":\"3\",\"total-records\":\"6\",\"total-files-size\":\"#Masked#\",\"total-d
[...]
+ current-snapshot-summary
{\"manifests-created\":\"4\",\"manifests-kept\":\"1\",\"manifests-replaced\":\"3\",\"added-data-files\":\"1\",\"deleted-data-files\":\"4\",\"removed-position-delete-files\":\"2\",\"removed-delete-files\":\"2\",\"added-records\":\"5\",\"deleted-records\":\"6\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"removed-position-deletes\":\"2\",\"changed-partition-count\":\"3\",\"total-records\":\"10\",\"total-files-size\":\"#Masked#\",\"total-
[...]
current-snapshot-timestamp-ms #Masked#
default-partition-spec
{\"spec-id\":1,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000}]}
format-version 2
#### A masked pattern was here ####
numFiles 2
numPartitions 2
- numRows 6
+ numRows 10
parquet.compression zstd
#### A masked pattern was here ####
rawDataSize 0
serialization.format 1
- snapshot-count 8
+ snapshot-count 12
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
totalSize #Masked#
@@ -367,6 +403,22 @@ POSTHOOK: query: insert into ice_orc VALUES
('fn11','ln11', 3), ('fn12','ln12',
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: ice_comp@ice_orc
+PREHOOK: query: insert into ice_orc VALUES ('fn19','ln19', 3)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: ice_comp@ice_orc
+POSTHOOK: query: insert into ice_orc VALUES ('fn19','ln19', 3)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: ice_comp@ice_orc
+PREHOOK: query: insert into ice_orc VALUES ('fn20','ln20', 3)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: ice_comp@ice_orc
+POSTHOOK: query: insert into ice_orc VALUES ('fn20','ln20', 3)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: ice_comp@ice_orc
PREHOOK: query: select * from ice_orc
PREHOOK: type: QUERY
PREHOOK: Input: ice_comp@ice_orc
@@ -379,7 +431,13 @@ fn1 ln1 1
fn10 ln10 3
fn11 ln11 3
fn12 ln12 3
+fn15 ln15 1
+fn16 ln16 1
+fn17 ln17 2
+fn18 ln18 2
+fn19 ln19 3
fn2 ln2 1
+fn20 ln20 3
fn3 ln3 1
fn5 ln5 2
fn6 ln6 2
@@ -410,17 +468,17 @@ Table Parameters:
compactor.threshold.target.size 4000
current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"}]}
current-snapshot-id #Masked#
- current-snapshot-summary
{\"manifests-created\":\"1\",\"manifests-kept\":\"2\",\"manifests-replaced\":\"0\",\"added-data-files\":\"1\",\"added-records\":\"2\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"10\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"4\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"}
+ current-snapshot-summary
{\"manifests-created\":\"1\",\"manifests-kept\":\"4\",\"manifests-replaced\":\"0\",\"added-data-files\":\"1\",\"added-records\":\"1\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"16\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"6\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"}
current-snapshot-timestamp-ms #Masked#
format-version 2
#### A masked pattern was here ####
- numFiles 4
- numRows 10
+ numFiles 6
+ numRows 16
parquet.compression zstd
#### A masked pattern was here ####
rawDataSize 0
serialization.format 1
- snapshot-count 10
+ snapshot-count 16
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
totalSize #Masked#
@@ -466,7 +524,13 @@ fn1 ln1 1
fn10 ln10 3
fn11 ln11 3
fn12 ln12 3
+fn15 ln15 1
+fn16 ln16 1
+fn17 ln17 2
+fn18 ln18 2
+fn19 ln19 3
fn2 ln2 1
+fn20 ln20 3
fn3 ln3 1
fn5 ln5 2
fn6 ln6 2
@@ -497,17 +561,17 @@ Table Parameters:
compactor.threshold.target.size 4000
current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"}]}
current-snapshot-id #Masked#
- current-snapshot-summary
{\"manifests-created\":\"4\",\"manifests-kept\":\"0\",\"manifests-replaced\":\"3\",\"added-data-files\":\"1\",\"deleted-data-files\":\"4\",\"added-records\":\"10\",\"deleted-records\":\"10\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"3\",\"total-records\":\"10\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-delete
[...]
+ current-snapshot-summary
{\"manifests-created\":\"6\",\"manifests-kept\":\"0\",\"manifests-replaced\":\"5\",\"added-data-files\":\"1\",\"deleted-data-files\":\"6\",\"added-records\":\"16\",\"deleted-records\":\"16\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"3\",\"total-records\":\"16\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-delete
[...]
current-snapshot-timestamp-ms #Masked#
format-version 2
#### A masked pattern was here ####
numFiles 1
- numRows 10
+ numRows 16
parquet.compression zstd
#### A masked pattern was here ####
rawDataSize 0
serialization.format 1
- snapshot-count 11
+ snapshot-count 17
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
totalSize #Masked#
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_unpartitioned.q.out
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_unpartitioned.q.out
index a1eace49222..b05d328fe1e 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_unpartitioned.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_unpartitioned.q.out
@@ -24,27 +24,59 @@ POSTHOOK: query: insert into ice_orc VALUES ('fn1','ln1')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc VALUES ('fn5','ln5'), ('fn7','ln7'),
('fn4','ln4'), ('fn6','ln6')
+PREHOOK: query: insert into ice_orc VALUES ('fn5','ln5')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc VALUES ('fn5','ln5'), ('fn7','ln7'),
('fn4','ln4'), ('fn6','ln6')
+POSTHOOK: query: insert into ice_orc VALUES ('fn5','ln5')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc VALUES ('fn3','ln3'), ('fn2','ln2')
+PREHOOK: query: insert into ice_orc VALUES ('fn7','ln7')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc VALUES ('fn3','ln3'), ('fn2','ln2')
+POSTHOOK: query: insert into ice_orc VALUES ('fn7','ln7')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: alter table ice_orc set tblproperties
('compactor.threshold.target.size'='2920',
'compactor.threshold.min.input.files'='2')
+PREHOOK: query: insert into ice_orc VALUES ('fn4','ln4')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc VALUES ('fn4','ln4')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc VALUES ('fn6','ln6')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc VALUES ('fn6','ln6')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc VALUES ('fn3','ln3')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc VALUES ('fn3','ln3')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc VALUES ('fn2','ln2')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc VALUES ('fn2','ln2')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: alter table ice_orc set tblproperties
('compactor.threshold.target.size'='4000',
'compactor.threshold.min.input.files'='2')
PREHOOK: type: ALTERTABLE_PROPERTIES
PREHOOK: Input: default@ice_orc
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: alter table ice_orc set tblproperties
('compactor.threshold.target.size'='2920',
'compactor.threshold.min.input.files'='2')
+POSTHOOK: query: alter table ice_orc set tblproperties
('compactor.threshold.target.size'='4000',
'compactor.threshold.min.input.files'='2')
POSTHOOK: type: ALTERTABLE_PROPERTIES
POSTHOOK: Input: default@ice_orc
POSTHOOK: Output: default@ice_orc
@@ -90,20 +122,20 @@ Table Parameters:
EXTERNAL TRUE
bucketing_version 2
compactor.threshold.min.input.files 2
- compactor.threshold.target.size 2920
+ compactor.threshold.target.size 4000
current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"}]}
current-snapshot-id #Masked#
- current-snapshot-summary
{\"manifests-created\":\"1\",\"manifests-kept\":\"2\",\"manifests-replaced\":\"0\",\"added-data-files\":\"1\",\"added-records\":\"2\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"7\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"3\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"}
+ current-snapshot-summary
{\"manifests-created\":\"1\",\"manifests-kept\":\"6\",\"manifests-replaced\":\"0\",\"added-data-files\":\"1\",\"added-records\":\"1\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"7\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"7\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"}
current-snapshot-timestamp-ms #Masked#
format-version 2
#### A masked pattern was here ####
- numFiles 3
+ numFiles 7
numRows 7
parquet.compression zstd
#### A masked pattern was here ####
rawDataSize 0
serialization.format 1
- snapshot-count 3
+ snapshot-count 7
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
totalSize #Masked#
@@ -173,10 +205,10 @@ Table Parameters:
EXTERNAL TRUE
bucketing_version 2
compactor.threshold.min.input.files 2
- compactor.threshold.target.size 2920
+ compactor.threshold.target.size 4000
current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"}]}
current-snapshot-id #Masked#
- current-snapshot-summary
{\"manifests-created\":\"4\",\"manifests-kept\":\"0\",\"manifests-replaced\":\"3\",\"added-data-files\":\"1\",\"deleted-data-files\":\"3\",\"added-records\":\"7\",\"deleted-records\":\"7\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"7\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\"
[...]
+ current-snapshot-summary
{\"manifests-created\":\"8\",\"manifests-kept\":\"0\",\"manifests-replaced\":\"7\",\"added-data-files\":\"1\",\"deleted-data-files\":\"7\",\"added-records\":\"7\",\"deleted-records\":\"7\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"7\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\"
[...]
current-snapshot-timestamp-ms #Masked#
format-version 2
#### A masked pattern was here ####
@@ -186,7 +218,7 @@ Table Parameters:
#### A masked pattern was here ####
rawDataSize 0
serialization.format 1
- snapshot-count 4
+ snapshot-count 8
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
totalSize #Masked#
@@ -270,7 +302,7 @@ Table Parameters:
EXTERNAL TRUE
bucketing_version 2
compactor.threshold.min.input.files 2
- compactor.threshold.target.size 2920
+ compactor.threshold.target.size 4000
current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"}]}
current-snapshot-id #Masked#
current-snapshot-summary
{\"manifests-created\":\"4\",\"manifests-kept\":\"0\",\"manifests-replaced\":\"3\",\"added-data-files\":\"1\",\"deleted-data-files\":\"2\",\"removed-position-delete-files\":\"1\",\"removed-delete-files\":\"1\",\"added-records\":\"7\",\"deleted-records\":\"8\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"removed-position-deletes\":\"1\",\"changed-partition-count\":\"1\",\"total-records\":\"7\",\"total-files-size\":\"#Masked#\",\"total-d
[...]
@@ -283,7 +315,7 @@ Table Parameters:
#### A masked pattern was here ####
rawDataSize 0
serialization.format 1
- snapshot-count 7
+ snapshot-count 11
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
totalSize #Masked#
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_row_lineage_compactions.q.out
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_row_lineage_compactions.q.out
index b4d8dcd592d..7f593a3a577 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_row_lineage_compactions.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_row_lineage_compactions.q.out
@@ -229,6 +229,22 @@ POSTHOOK: query: insert into part_tbl values (5,'p5', 10)
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: ice_comp_all@part_tbl
+PREHOOK: query: insert into part_tbl values (5,'p5', 10)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: ice_comp_all@part_tbl
+POSTHOOK: query: insert into part_tbl values (5,'p5', 10)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: ice_comp_all@part_tbl
+PREHOOK: query: insert into part_tbl values (6,'p6', 20)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: ice_comp_all@part_tbl
+POSTHOOK: query: insert into part_tbl values (6,'p6', 20)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: ice_comp_all@part_tbl
PREHOOK: query: insert into part_tbl values (6,'p6', 20)
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
@@ -254,7 +270,9 @@ POSTHOOK: Input: ice_comp_all@part_tbl
3 p3 20 2 3
4 p4 20 3 4
5 p5 10 11 9
-6 p6 20 16 10
+5 p5 10 16 10
+6 p6 20 17 11
+6 p6 20 18 12
PREHOOK: query: alter table part_tbl compact 'minor' and wait pool 'iceberg'
PREHOOK: type: ALTERTABLE_COMPACT
PREHOOK: Input: ice_comp_all@part_tbl
@@ -280,7 +298,9 @@ POSTHOOK: Input: ice_comp_all@part_tbl
3 p3 20 2 3
4 p4 20 3 4
5 p5 10 11 9
-6 p6 20 16 10
+5 p5 10 16 10
+6 p6 20 17 11
+6 p6 20 18 12
PREHOOK: query: show compactions
PREHOOK: type: SHOW COMPACTIONS
POSTHOOK: query: show compactions
