This is an automated email from the ASF dual-hosted git repository.
simhadrig pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 90d48dfa1a0 HIVE-28524: Iceberg: Major QB Compaction add sort order
support (#5456)(Dmitriy Fingerman, reviewed by Simhadri Govindappa)
90d48dfa1a0 is described below
commit 90d48dfa1a05b0105601b00581b9216e31f577c2
Author: Dmitriy Fingerman <[email protected]>
AuthorDate: Thu Oct 10 07:16:16 2024 -0400
HIVE-28524: Iceberg: Major QB Compaction add sort order support
(#5456)(Dmitriy Fingerman, reviewed by Simhadri Govindappa)
Co-authored-by: Dmitriy Fingerman <[email protected]>
---
.../compaction/IcebergMajorQueryCompactor.java | 17 +-
...major_compaction_partition_evolution_ordered.q} | 62 +++--
.../iceberg_major_compaction_unpartitioned.q | 2 +
...eberg_major_compaction_unpartitioned_ordered.q} | 34 +--
...r_compaction_partition_evolution_ordered.q.out} | 289 +++++++++++----------
.../iceberg_major_compaction_unpartitioned.q.out | 20 ++
...g_major_compaction_unpartitioned_ordered.q.out} | 202 +++++---------
.../test/resources/testconfiguration.properties | 5 +-
.../hadoop/hive/ql/parse/AlterClauseParser.g | 4 +-
.../hive/ql/parse/TestParseOptimizeTable.java | 67 +++--
.../hive/ql/txn/compactor/CompactorUtil.java | 1 +
11 files changed, 345 insertions(+), 358 deletions(-)
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergMajorQueryCompactor.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergMajorQueryCompactor.java
index d200eafd1b9..e334f631e32 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergMajorQueryCompactor.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergMajorQueryCompactor.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.txn.entities.CompactionInfo;
import org.apache.hadoop.hive.ql.Context.RewritePolicy;
import org.apache.hadoop.hive.ql.DriverUtils;
import org.apache.hadoop.hive.ql.ErrorMsg;
@@ -54,25 +55,27 @@ public class IcebergMajorQueryCompactor extends
QueryCompactor {
LOG.debug("Initiating compaction for the {} table", compactTableName);
HiveConf conf = new HiveConf(context.getConf());
- String partSpec = context.getCompactionInfo().partName;
+ CompactionInfo ci = context.getCompactionInfo();
+ String partSpec = ci.partName;
org.apache.hadoop.hive.ql.metadata.Table table =
Hive.get(conf).getTable(context.getTable().getDbName(),
context.getTable().getTableName());
Table icebergTable = IcebergTableUtil.getTable(conf, table.getTTable());
String compactionQuery;
+ String orderBy = ci.orderByClause == null ? "" : ci.orderByClause;
if (partSpec == null) {
if (!icebergTable.spec().isPartitioned()) {
HiveConf.setVar(conf, ConfVars.REWRITE_POLICY,
RewritePolicy.FULL_TABLE.name());
- compactionQuery = String.format("insert overwrite table %s select *
from %<s", compactTableName);
+ compactionQuery = String.format("insert overwrite table %s select *
from %<s %2$s", compactTableName, orderBy);
} else if (icebergTable.specs().size() > 1) {
// Compacting partitions of old partition specs on a partitioned table
with partition evolution
HiveConf.setVar(conf, ConfVars.REWRITE_POLICY,
RewritePolicy.PARTITION.name());
// A single filter on a virtual column causes errors during
compilation,
// added another filter on file_path as a workaround.
compactionQuery = String.format("insert overwrite table %1$s select *
from %1$s " +
- "where %2$s != %3$d and %4$s is not null",
+ "where %2$s != %3$d and %4$s is not null %5$s",
compactTableName, VirtualColumn.PARTITION_SPEC_ID.getName(),
icebergTable.spec().specId(),
- VirtualColumn.FILE_PATH.getName());
+ VirtualColumn.FILE_PATH.getName(), orderBy);
} else {
// Partitioned table without partition evolution with partition spec
as null in the compaction request - this
// code branch is not supposed to be reachable
@@ -87,11 +90,11 @@ public class IcebergMajorQueryCompactor extends
QueryCompactor {
Warehouse.makeSpecFromName(partSpecMap, new Path(partSpec), null);
compactionQuery = String.format("insert overwrite table %1$s select *
from %1$s where %2$s=%3$d " +
- "and %4$s is not null", compactTableName,
VirtualColumn.PARTITION_HASH.getName(), partitionHash,
- VirtualColumn.FILE_PATH.getName());
+ "and %4$s is not null %5$s", compactTableName,
VirtualColumn.PARTITION_HASH.getName(), partitionHash,
+ VirtualColumn.FILE_PATH.getName(), orderBy);
}
- SessionState sessionState = setupQueryCompactionSession(conf,
context.getCompactionInfo(), tblProperties);
+ SessionState sessionState = setupQueryCompactionSession(conf, ci,
tblProperties);
String compactionTarget = "table " +
HiveUtils.unparseIdentifier(compactTableName) +
(partSpec != null ? ", partition " +
HiveUtils.unparseIdentifier(partSpec) : "");
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_unpartitioned.q
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_ordered.q
similarity index 54%
copy from
iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_unpartitioned.q
copy to
iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_ordered.q
index 2324ab9dfee..da81a842abe 100644
---
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_unpartitioned.q
+++
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_ordered.q
@@ -1,4 +1,3 @@
--- SORT_QUERY_RESULTS
-- Mask neededVirtualColumns due to non-strict order
--! qt:replace:/(\s+neededVirtualColumns:\s)(.*)/$1#Masked#/
-- Mask random uuid
@@ -11,7 +10,6 @@
--! qt:replace:/(\S\"total-files-size\\\":\\\")(\d+)(\\\")/$1#Masked#$3/
-- Mask current-snapshot-timestamp-ms
--! qt:replace:/(\s+current-snapshot-timestamp-ms\s+)\S+(\s*)/$1#Masked#$2/
--- Mask the enqueue time which is based on current time
--!
qt:replace:/(MAJOR\s+succeeded\s+)[a-zA-Z0-9\-\.\s+]+(\s+manual)/$1#Masked#$2/
-- Mask compaction id as they will be allocated in parallel threads
--! qt:replace:/^[0-9]/#Masked#/
@@ -26,35 +24,47 @@ set hive.optimize.shared.work.merge.ts.schema=true;
create table ice_orc (
first_name string,
- last_name string
+ last_name string,
+ dept_id bigint,
+ team_id bigint
)
+partitioned by (company_id bigint)
stored by iceberg stored as orc
tblproperties ('format-version'='2');
-insert into ice_orc VALUES ('fn1','ln1');
-insert into ice_orc VALUES ('fn2','ln2');
-insert into ice_orc VALUES ('fn3','ln3');
-insert into ice_orc VALUES ('fn4','ln4');
-insert into ice_orc VALUES ('fn5','ln5');
-insert into ice_orc VALUES ('fn6','ln6');
-insert into ice_orc VALUES ('fn7','ln7');
-
-update ice_orc set last_name = 'ln1a' where first_name='fn1';
-update ice_orc set last_name = 'ln2a' where first_name='fn2';
-update ice_orc set last_name = 'ln3a' where first_name='fn3';
-update ice_orc set last_name = 'ln4a' where first_name='fn4';
-update ice_orc set last_name = 'ln5a' where first_name='fn5';
-update ice_orc set last_name = 'ln6a' where first_name='fn6';
-update ice_orc set last_name = 'ln7a' where first_name='fn7';
-
-delete from ice_orc where last_name in ('ln5a', 'ln6a', 'ln7a');
-
-select * from ice_orc;
+insert into ice_orc VALUES
+('fn1','ln1', 1, 10, 100),
+('fn2','ln2', 1, 10, 100),
+('fn3','ln3', 1, 11, 100),
+('fn4','ln4', 1, 11, 100);
+
+alter table ice_orc set partition spec(dept_id);
+
+insert into ice_orc VALUES
+('fn5','ln5', 2, 20, 101),
+('fn6','ln6', 2, 20, 101),
+('fn7','ln7', 2, 20, 101),
+('fn8','ln8', 2, 20, 101);
+
+insert into ice_orc VALUES
+('fn9', 'ln9', 3, 20, 101),
+('fn10','ln10', 3, 20, 101),
+('fn11','ln11', 3, 20, 101),
+('fn12','ln12', 3, 20, 101);
+
+select * from ice_orc where company_id = 100;
+select * from ice_orc where dept_id = 2;
+select * from ice_orc where dept_id = 3;
describe formatted ice_orc;
-explain alter table ice_orc COMPACT 'major' and wait;
-alter table ice_orc COMPACT 'major' and wait;
+explain alter table ice_orc COMPACT 'major' and wait order by first_name desc;
+explain optimize table ice_orc rewrite data order by first_name desc;
+
+alter table ice_orc COMPACT 'major' and wait order by first_name desc;
+
+select * from ice_orc where company_id = 100;
+select * from ice_orc where dept_id = 2;
+select * from ice_orc where dept_id = 3;
-select * from ice_orc;
describe formatted ice_orc;
-show compactions;
\ No newline at end of file
+show compactions order by 'partition';
\ No newline at end of file
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_unpartitioned.q
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_unpartitioned.q
index 2324ab9dfee..8268be8cd3f 100644
---
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_unpartitioned.q
+++
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_unpartitioned.q
@@ -53,6 +53,8 @@ select * from ice_orc;
describe formatted ice_orc;
explain alter table ice_orc COMPACT 'major' and wait;
+explain optimize table ice_orc rewrite data;
+
alter table ice_orc COMPACT 'major' and wait;
select * from ice_orc;
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_optimize_table_unpartitioned.q
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_unpartitioned_ordered.q
similarity index 57%
rename from
iceberg/iceberg-handler/src/test/queries/positive/iceberg_optimize_table_unpartitioned.q
rename to
iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_unpartitioned_ordered.q
index 7efa9732751..26ccab22847 100644
---
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_optimize_table_unpartitioned.q
+++
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_unpartitioned_ordered.q
@@ -1,8 +1,5 @@
--- SORT_QUERY_RESULTS
-- Mask neededVirtualColumns due to non-strict order
--! qt:replace:/(\s+neededVirtualColumns:\s)(.*)/$1#Masked#/
--- Mask the totalSize value as it can have slight variability, causing test
flakiness
---! qt:replace:/(\s+totalSize\s+)\S+(\s+)/$1#Masked#$2/
-- Mask random uuid
--! qt:replace:/(\s+uuid\s+)\S+(\s*)/$1#Masked#$2/
-- Mask a random snapshot id
@@ -33,29 +30,24 @@ create table ice_orc (
stored by iceberg stored as orc
tblproperties ('format-version'='2');
-insert into ice_orc VALUES ('fn1','ln1');
-insert into ice_orc VALUES ('fn2','ln2');
-insert into ice_orc VALUES ('fn3','ln3');
-insert into ice_orc VALUES ('fn4','ln4');
-insert into ice_orc VALUES ('fn5','ln5');
-insert into ice_orc VALUES ('fn6','ln6');
-insert into ice_orc VALUES ('fn7','ln7');
+insert into ice_orc VALUES
+('fn1','ln1'),
+('fn2','ln2'),
+('fn3','ln3'),
+('fn4','ln4'),
+('fn5','ln5'),
+('fn6','ln6'),
+('fn7','ln7');
-update ice_orc set last_name = 'ln1a' where first_name='fn1';
-update ice_orc set last_name = 'ln2a' where first_name='fn2';
-update ice_orc set last_name = 'ln3a' where first_name='fn3';
-update ice_orc set last_name = 'ln4a' where first_name='fn4';
-update ice_orc set last_name = 'ln5a' where first_name='fn5';
-update ice_orc set last_name = 'ln6a' where first_name='fn6';
-update ice_orc set last_name = 'ln7a' where first_name='fn7';
-
-delete from ice_orc where last_name in ('ln5a', 'ln6a', 'ln7a');
+delete from ice_orc where last_name in ('ln5', 'ln6', 'ln7');
select * from ice_orc;
describe formatted ice_orc;
-explain optimize table ice_orc rewrite data;
-optimize table ice_orc rewrite data;
+explain alter table ice_orc COMPACT 'major' and wait order by last_name desc;
+explain optimize table ice_orc rewrite data order by last_name desc;
+
+alter table ice_orc COMPACT 'major' and wait order by last_name desc;
select * from ice_orc;
describe formatted ice_orc;
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_unpartitioned.q.out
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out
similarity index 53%
copy from
iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_unpartitioned.q.out
copy to
iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out
index fc5c9d2a6c4..3d1565ff9e5 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_unpartitioned.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out
@@ -1,7 +1,10 @@
PREHOOK: query: create table ice_orc (
first_name string,
- last_name string
+ last_name string,
+ dept_id bigint,
+ team_id bigint
)
+partitioned by (company_id bigint)
stored by iceberg stored as orc
tblproperties ('format-version'='2')
PREHOOK: type: CREATETABLE
@@ -9,159 +12,107 @@ PREHOOK: Output: database:default
PREHOOK: Output: default@ice_orc
POSTHOOK: query: create table ice_orc (
first_name string,
- last_name string
+ last_name string,
+ dept_id bigint,
+ team_id bigint
)
+partitioned by (company_id bigint)
stored by iceberg stored as orc
tblproperties ('format-version'='2')
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc VALUES ('fn1','ln1')
+PREHOOK: query: insert into ice_orc VALUES
+('fn1','ln1', 1, 10, 100),
+('fn2','ln2', 1, 10, 100),
+('fn3','ln3', 1, 11, 100),
+('fn4','ln4', 1, 11, 100)
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc VALUES ('fn1','ln1')
+POSTHOOK: query: insert into ice_orc VALUES
+('fn1','ln1', 1, 10, 100),
+('fn2','ln2', 1, 10, 100),
+('fn3','ln3', 1, 11, 100),
+('fn4','ln4', 1, 11, 100)
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc VALUES ('fn2','ln2')
-PREHOOK: type: QUERY
-PREHOOK: Input: _dummy_database@_dummy_table
-PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc VALUES ('fn2','ln2')
-POSTHOOK: type: QUERY
-POSTHOOK: Input: _dummy_database@_dummy_table
-POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc VALUES ('fn3','ln3')
-PREHOOK: type: QUERY
-PREHOOK: Input: _dummy_database@_dummy_table
-PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc VALUES ('fn3','ln3')
-POSTHOOK: type: QUERY
-POSTHOOK: Input: _dummy_database@_dummy_table
-POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc VALUES ('fn4','ln4')
-PREHOOK: type: QUERY
-PREHOOK: Input: _dummy_database@_dummy_table
-PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc VALUES ('fn4','ln4')
-POSTHOOK: type: QUERY
-POSTHOOK: Input: _dummy_database@_dummy_table
-POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc VALUES ('fn5','ln5')
-PREHOOK: type: QUERY
-PREHOOK: Input: _dummy_database@_dummy_table
-PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc VALUES ('fn5','ln5')
-POSTHOOK: type: QUERY
-POSTHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: query: alter table ice_orc set partition spec(dept_id)
+PREHOOK: type: ALTERTABLE_SETPARTSPEC
+PREHOOK: Input: default@ice_orc
+POSTHOOK: query: alter table ice_orc set partition spec(dept_id)
+POSTHOOK: type: ALTERTABLE_SETPARTSPEC
+POSTHOOK: Input: default@ice_orc
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc VALUES ('fn6','ln6')
+PREHOOK: query: insert into ice_orc VALUES
+('fn5','ln5', 2, 20, 101),
+('fn6','ln6', 2, 20, 101),
+('fn7','ln7', 2, 20, 101),
+('fn8','ln8', 2, 20, 101)
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc VALUES ('fn6','ln6')
+POSTHOOK: query: insert into ice_orc VALUES
+('fn5','ln5', 2, 20, 101),
+('fn6','ln6', 2, 20, 101),
+('fn7','ln7', 2, 20, 101),
+('fn8','ln8', 2, 20, 101)
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc VALUES ('fn7','ln7')
+PREHOOK: query: insert into ice_orc VALUES
+('fn9', 'ln9', 3, 20, 101),
+('fn10','ln10', 3, 20, 101),
+('fn11','ln11', 3, 20, 101),
+('fn12','ln12', 3, 20, 101)
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc VALUES ('fn7','ln7')
+POSTHOOK: query: insert into ice_orc VALUES
+('fn9', 'ln9', 3, 20, 101),
+('fn10','ln10', 3, 20, 101),
+('fn11','ln11', 3, 20, 101),
+('fn12','ln12', 3, 20, 101)
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: update ice_orc set last_name = 'ln1a' where first_name='fn1'
-PREHOOK: type: QUERY
-PREHOOK: Input: default@ice_orc
-PREHOOK: Output: default@ice_orc
-PREHOOK: Output: default@ice_orc
-POSTHOOK: query: update ice_orc set last_name = 'ln1a' where first_name='fn1'
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-PREHOOK: query: update ice_orc set last_name = 'ln2a' where first_name='fn2'
-PREHOOK: type: QUERY
-PREHOOK: Input: default@ice_orc
-PREHOOK: Output: default@ice_orc
-PREHOOK: Output: default@ice_orc
-POSTHOOK: query: update ice_orc set last_name = 'ln2a' where first_name='fn2'
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-PREHOOK: query: update ice_orc set last_name = 'ln3a' where first_name='fn3'
-PREHOOK: type: QUERY
-PREHOOK: Input: default@ice_orc
-PREHOOK: Output: default@ice_orc
-PREHOOK: Output: default@ice_orc
-POSTHOOK: query: update ice_orc set last_name = 'ln3a' where first_name='fn3'
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-PREHOOK: query: update ice_orc set last_name = 'ln4a' where first_name='fn4'
+PREHOOK: query: select * from ice_orc where company_id = 100
PREHOOK: type: QUERY
PREHOOK: Input: default@ice_orc
-PREHOOK: Output: default@ice_orc
-PREHOOK: Output: default@ice_orc
-POSTHOOK: query: update ice_orc set last_name = 'ln4a' where first_name='fn4'
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-PREHOOK: query: update ice_orc set last_name = 'ln5a' where first_name='fn5'
-PREHOOK: type: QUERY
-PREHOOK: Input: default@ice_orc
-PREHOOK: Output: default@ice_orc
-PREHOOK: Output: default@ice_orc
-POSTHOOK: query: update ice_orc set last_name = 'ln5a' where first_name='fn5'
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-PREHOOK: query: update ice_orc set last_name = 'ln6a' where first_name='fn6'
-PREHOOK: type: QUERY
-PREHOOK: Input: default@ice_orc
-PREHOOK: Output: default@ice_orc
-PREHOOK: Output: default@ice_orc
-POSTHOOK: query: update ice_orc set last_name = 'ln6a' where first_name='fn6'
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-PREHOOK: query: update ice_orc set last_name = 'ln7a' where first_name='fn7'
-PREHOOK: type: QUERY
-PREHOOK: Input: default@ice_orc
-PREHOOK: Output: default@ice_orc
-PREHOOK: Output: default@ice_orc
-POSTHOOK: query: update ice_orc set last_name = 'ln7a' where first_name='fn7'
+#### A masked pattern was here ####
+POSTHOOK: query: select * from ice_orc where company_id = 100
POSTHOOK: type: QUERY
POSTHOOK: Input: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-PREHOOK: query: delete from ice_orc where last_name in ('ln5a', 'ln6a', 'ln7a')
+#### A masked pattern was here ####
+fn1 ln1 1 10 100
+fn2 ln2 1 10 100
+fn3 ln3 1 11 100
+fn4 ln4 1 11 100
+PREHOOK: query: select * from ice_orc where dept_id = 2
PREHOOK: type: QUERY
PREHOOK: Input: default@ice_orc
#### A masked pattern was here ####
-POSTHOOK: query: delete from ice_orc where last_name in ('ln5a', 'ln6a',
'ln7a')
+POSTHOOK: query: select * from ice_orc where dept_id = 2
POSTHOOK: type: QUERY
POSTHOOK: Input: default@ice_orc
#### A masked pattern was here ####
-PREHOOK: query: select * from ice_orc
+fn5 ln5 2 20 101
+fn6 ln6 2 20 101
+fn7 ln7 2 20 101
+fn8 ln8 2 20 101
+PREHOOK: query: select * from ice_orc where dept_id = 3
PREHOOK: type: QUERY
PREHOOK: Input: default@ice_orc
#### A masked pattern was here ####
-POSTHOOK: query: select * from ice_orc
+POSTHOOK: query: select * from ice_orc where dept_id = 3
POSTHOOK: type: QUERY
POSTHOOK: Input: default@ice_orc
#### A masked pattern was here ####
-fn1 ln1a
-fn2 ln2a
-fn3 ln3a
-fn4 ln4a
+fn9 ln9 3 20 101
+fn10 ln10 3 20 101
+fn11 ln11 3 20 101
+fn12 ln12 3 20 101
PREHOOK: query: describe formatted ice_orc
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_orc
@@ -171,6 +122,13 @@ POSTHOOK: Input: default@ice_orc
# col_name data_type comment
first_name string
last_name string
+dept_id bigint
+team_id bigint
+company_id bigint
+
+# Partition Transform Information
+# col_name transform_type
+dept_id IDENTITY
# Detailed Table Information
Database: default
@@ -179,23 +137,24 @@ Retention: 0
#### A masked pattern was here ####
Table Type: EXTERNAL_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE
{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"company_id\":\"true\",\"dept_id\":\"true\",\"first_name\":\"true\",\"last_name\":\"true\",\"team_id\":\"true\"}}
EXTERNAL TRUE
bucketing_version 2
- current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"}]}
+ current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]}
current-snapshot-id #Masked#
- current-snapshot-summary
{\"deleted-data-files\":\"3\",\"deleted-records\":\"3\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"11\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"11\",\"total-delete-files\":\"7\",\"total-position-deletes\":\"7\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"}
+ current-snapshot-summary
{\"added-data-files\":\"1\",\"added-records\":\"4\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"12\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"3\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"}
current-snapshot-timestamp-ms #Masked#
+ default-partition-spec
{\"spec-id\":1,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1001}]}
format-version 2
iceberg.orc.files.only true
#### A masked pattern was here ####
- numFiles 11
- numRows 11
+ numFiles 3
+ numRows 12
parquet.compression zstd
#### A masked pattern was here ####
rawDataSize 0
serialization.format 1
- snapshot-count 15
+ snapshot-count 3
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
totalSize #Masked#
@@ -212,11 +171,11 @@ InputFormat:
org.apache.iceberg.mr.hive.HiveIcebergInputFormat
OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
Compressed: No
Sort Columns: []
-PREHOOK: query: explain alter table ice_orc COMPACT 'major' and wait
+PREHOOK: query: explain alter table ice_orc COMPACT 'major' and wait order by
first_name desc
PREHOOK: type: ALTERTABLE_COMPACT
PREHOOK: Input: default@ice_orc
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: explain alter table ice_orc COMPACT 'major' and wait
+POSTHOOK: query: explain alter table ice_orc COMPACT 'major' and wait order by
first_name desc
POSTHOOK: type: ALTERTABLE_COMPACT
POSTHOOK: Input: default@ice_orc
POSTHOOK: Output: default@ice_orc
@@ -229,29 +188,75 @@ STAGE PLANS:
compaction type: major
table name: default.ice_orc
numberOfBuckets: 0
+ order by: order by first_name desc
table name: default.ice_orc
blocking: true
-PREHOOK: query: alter table ice_orc COMPACT 'major' and wait
+PREHOOK: query: explain optimize table ice_orc rewrite data order by
first_name desc
PREHOOK: type: ALTERTABLE_COMPACT
PREHOOK: Input: default@ice_orc
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: alter table ice_orc COMPACT 'major' and wait
+POSTHOOK: query: explain optimize table ice_orc rewrite data order by
first_name desc
POSTHOOK: type: ALTERTABLE_COMPACT
POSTHOOK: Input: default@ice_orc
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: select * from ice_orc
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Compact
+ compaction type: major
+ table name: default.ice_orc
+ numberOfBuckets: 0
+ order by: order by first_name desc
+ table name: default.ice_orc
+ blocking: true
+
+PREHOOK: query: alter table ice_orc COMPACT 'major' and wait order by
first_name desc
+PREHOOK: type: ALTERTABLE_COMPACT
+PREHOOK: Input: default@ice_orc
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: alter table ice_orc COMPACT 'major' and wait order by
first_name desc
+POSTHOOK: type: ALTERTABLE_COMPACT
+POSTHOOK: Input: default@ice_orc
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: select * from ice_orc where company_id = 100
PREHOOK: type: QUERY
PREHOOK: Input: default@ice_orc
#### A masked pattern was here ####
-POSTHOOK: query: select * from ice_orc
+POSTHOOK: query: select * from ice_orc where company_id = 100
POSTHOOK: type: QUERY
POSTHOOK: Input: default@ice_orc
#### A masked pattern was here ####
-fn1 ln1a
-fn2 ln2a
-fn3 ln3a
-fn4 ln4a
+fn4 ln4 1 11 100
+fn3 ln3 1 11 100
+fn2 ln2 1 10 100
+fn1 ln1 1 10 100
+PREHOOK: query: select * from ice_orc where dept_id = 2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_orc
+#### A masked pattern was here ####
+POSTHOOK: query: select * from ice_orc where dept_id = 2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_orc
+#### A masked pattern was here ####
+fn8 ln8 2 20 101
+fn7 ln7 2 20 101
+fn6 ln6 2 20 101
+fn5 ln5 2 20 101
+PREHOOK: query: select * from ice_orc where dept_id = 3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_orc
+#### A masked pattern was here ####
+POSTHOOK: query: select * from ice_orc where dept_id = 3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_orc
+#### A masked pattern was here ####
+fn9 ln9 3 20 101
+fn12 ln12 3 20 101
+fn11 ln11 3 20 101
+fn10 ln10 3 20 101
PREHOOK: query: describe formatted ice_orc
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_orc
@@ -261,6 +266,13 @@ POSTHOOK: Input: default@ice_orc
# col_name data_type comment
first_name string
last_name string
+dept_id bigint
+team_id bigint
+company_id bigint
+
+# Partition Transform Information
+# col_name transform_type
+dept_id IDENTITY
# Detailed Table Information
Database: default
@@ -269,23 +281,24 @@ Retention: 0
#### A masked pattern was here ####
Table Type: EXTERNAL_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE
{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"first_name\":\"true\",\"last_name\":\"true\"}}
+ COLUMN_STATS_ACCURATE
{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"company_id\":\"true\",\"dept_id\":\"true\",\"first_name\":\"true\",\"last_name\":\"true\",\"team_id\":\"true\"}}
EXTERNAL TRUE
bucketing_version 2
- current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"}]}
+ current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]}
current-snapshot-id #Masked#
- current-snapshot-summary
{\"added-data-files\":\"1\",\"deleted-data-files\":\"11\",\"removed-position-delete-files\":\"7\",\"removed-delete-files\":\"7\",\"added-records\":\"4\",\"deleted-records\":\"11\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"removed-position-deletes\":\"7\",\"changed-partition-count\":\"1\",\"total-records\":\"4\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",
[...]
+ current-snapshot-summary
{\"added-data-files\":\"1\",\"deleted-data-files\":\"1\",\"added-records\":\"4\",\"deleted-records\":\"4\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"2\",\"total-records\":\"12\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"3\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"}
current-snapshot-timestamp-ms #Masked#
+ default-partition-spec
{\"spec-id\":1,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1001}]}
format-version 2
iceberg.orc.files.only true
#### A masked pattern was here ####
- numFiles 1
- numRows 4
+ numFiles 3
+ numRows 12
parquet.compression zstd
#### A masked pattern was here ####
rawDataSize 0
serialization.format 1
- snapshot-count 16
+ snapshot-count 6
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
totalSize #Masked#
@@ -302,9 +315,11 @@ InputFormat:
org.apache.iceberg.mr.hive.HiveIcebergInputFormat
OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
Compressed: No
Sort Columns: []
-PREHOOK: query: show compactions
+PREHOOK: query: show compactions order by 'partition'
PREHOOK: type: SHOW COMPACTIONS
-POSTHOOK: query: show compactions
+POSTHOOK: query: show compactions order by 'partition'
POSTHOOK: type: SHOW COMPACTIONS
CompactionId Database Table Partition Type State Worker
host Worker Enqueue Time Start Time Duration(ms) HadoopJobId
Error message Initiator host Initiator Pool name TxnId Next
TxnId Commit Time Highest WriteId
+#Masked# default ice_orc dept_id=2 MAJOR succeeded
#Masked# manual default 0 0 0 ---
+#Masked# default ice_orc dept_id=3 MAJOR succeeded
#Masked# manual default 0 0 0 ---
#Masked# default ice_orc --- MAJOR succeeded #Masked#
manual default 0 0 0 ---
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_unpartitioned.q.out
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_unpartitioned.q.out
index fc5c9d2a6c4..5e25dfd1ca3 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_unpartitioned.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_unpartitioned.q.out
@@ -232,6 +232,26 @@ STAGE PLANS:
table name: default.ice_orc
blocking: true
+PREHOOK: query: explain optimize table ice_orc rewrite data
+PREHOOK: type: ALTERTABLE_COMPACT
+PREHOOK: Input: default@ice_orc
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: explain optimize table ice_orc rewrite data
+POSTHOOK: type: ALTERTABLE_COMPACT
+POSTHOOK: Input: default@ice_orc
+POSTHOOK: Output: default@ice_orc
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Compact
+ compaction type: major
+ table name: default.ice_orc
+ numberOfBuckets: 0
+ table name: default.ice_orc
+ blocking: true
+
PREHOOK: query: alter table ice_orc COMPACT 'major' and wait
PREHOOK: type: ALTERTABLE_COMPACT
PREHOOK: Input: default@ice_orc
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_optimize_table_unpartitioned.q.out
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_unpartitioned_ordered.q.out
similarity index 52%
rename from
iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_optimize_table_unpartitioned.q.out
rename to
iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_unpartitioned_ordered.q.out
index a544f0537d0..5c3edbc8dd1 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_optimize_table_unpartitioned.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_unpartitioned_ordered.q.out
@@ -16,140 +16,36 @@ tblproperties ('format-version'='2')
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc VALUES ('fn1','ln1')
+PREHOOK: query: insert into ice_orc VALUES
+('fn1','ln1'),
+('fn2','ln2'),
+('fn3','ln3'),
+('fn4','ln4'),
+('fn5','ln5'),
+('fn6','ln6'),
+('fn7','ln7')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc VALUES ('fn1','ln1')
+POSTHOOK: query: insert into ice_orc VALUES
+('fn1','ln1'),
+('fn2','ln2'),
+('fn3','ln3'),
+('fn4','ln4'),
+('fn5','ln5'),
+('fn6','ln6'),
+('fn7','ln7')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc VALUES ('fn2','ln2')
-PREHOOK: type: QUERY
-PREHOOK: Input: _dummy_database@_dummy_table
-PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc VALUES ('fn2','ln2')
-POSTHOOK: type: QUERY
-POSTHOOK: Input: _dummy_database@_dummy_table
-POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc VALUES ('fn3','ln3')
-PREHOOK: type: QUERY
-PREHOOK: Input: _dummy_database@_dummy_table
-PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc VALUES ('fn3','ln3')
-POSTHOOK: type: QUERY
-POSTHOOK: Input: _dummy_database@_dummy_table
-POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc VALUES ('fn4','ln4')
-PREHOOK: type: QUERY
-PREHOOK: Input: _dummy_database@_dummy_table
-PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc VALUES ('fn4','ln4')
-POSTHOOK: type: QUERY
-POSTHOOK: Input: _dummy_database@_dummy_table
-POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc VALUES ('fn5','ln5')
-PREHOOK: type: QUERY
-PREHOOK: Input: _dummy_database@_dummy_table
-PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc VALUES ('fn5','ln5')
-POSTHOOK: type: QUERY
-POSTHOOK: Input: _dummy_database@_dummy_table
-POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc VALUES ('fn6','ln6')
-PREHOOK: type: QUERY
-PREHOOK: Input: _dummy_database@_dummy_table
-PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc VALUES ('fn6','ln6')
-POSTHOOK: type: QUERY
-POSTHOOK: Input: _dummy_database@_dummy_table
-POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc VALUES ('fn7','ln7')
-PREHOOK: type: QUERY
-PREHOOK: Input: _dummy_database@_dummy_table
-PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc VALUES ('fn7','ln7')
-POSTHOOK: type: QUERY
-POSTHOOK: Input: _dummy_database@_dummy_table
-POSTHOOK: Output: default@ice_orc
-PREHOOK: query: update ice_orc set last_name = 'ln1a' where first_name='fn1'
-PREHOOK: type: QUERY
-PREHOOK: Input: default@ice_orc
-PREHOOK: Output: default@ice_orc
-PREHOOK: Output: default@ice_orc
-POSTHOOK: query: update ice_orc set last_name = 'ln1a' where first_name='fn1'
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-PREHOOK: query: update ice_orc set last_name = 'ln2a' where first_name='fn2'
-PREHOOK: type: QUERY
-PREHOOK: Input: default@ice_orc
-PREHOOK: Output: default@ice_orc
-PREHOOK: Output: default@ice_orc
-POSTHOOK: query: update ice_orc set last_name = 'ln2a' where first_name='fn2'
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-PREHOOK: query: update ice_orc set last_name = 'ln3a' where first_name='fn3'
-PREHOOK: type: QUERY
-PREHOOK: Input: default@ice_orc
-PREHOOK: Output: default@ice_orc
-PREHOOK: Output: default@ice_orc
-POSTHOOK: query: update ice_orc set last_name = 'ln3a' where first_name='fn3'
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-PREHOOK: query: update ice_orc set last_name = 'ln4a' where first_name='fn4'
-PREHOOK: type: QUERY
-PREHOOK: Input: default@ice_orc
-PREHOOK: Output: default@ice_orc
-PREHOOK: Output: default@ice_orc
-POSTHOOK: query: update ice_orc set last_name = 'ln4a' where first_name='fn4'
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-PREHOOK: query: update ice_orc set last_name = 'ln5a' where first_name='fn5'
-PREHOOK: type: QUERY
-PREHOOK: Input: default@ice_orc
-PREHOOK: Output: default@ice_orc
-PREHOOK: Output: default@ice_orc
-POSTHOOK: query: update ice_orc set last_name = 'ln5a' where first_name='fn5'
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-PREHOOK: query: update ice_orc set last_name = 'ln6a' where first_name='fn6'
-PREHOOK: type: QUERY
-PREHOOK: Input: default@ice_orc
-PREHOOK: Output: default@ice_orc
-PREHOOK: Output: default@ice_orc
-POSTHOOK: query: update ice_orc set last_name = 'ln6a' where first_name='fn6'
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-PREHOOK: query: update ice_orc set last_name = 'ln7a' where first_name='fn7'
+PREHOOK: query: delete from ice_orc where last_name in ('ln5', 'ln6', 'ln7')
PREHOOK: type: QUERY
PREHOOK: Input: default@ice_orc
PREHOOK: Output: default@ice_orc
-PREHOOK: Output: default@ice_orc
-POSTHOOK: query: update ice_orc set last_name = 'ln7a' where first_name='fn7'
+POSTHOOK: query: delete from ice_orc where last_name in ('ln5', 'ln6', 'ln7')
POSTHOOK: type: QUERY
POSTHOOK: Input: default@ice_orc
POSTHOOK: Output: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-PREHOOK: query: delete from ice_orc where last_name in ('ln5a', 'ln6a', 'ln7a')
-PREHOOK: type: QUERY
-PREHOOK: Input: default@ice_orc
-#### A masked pattern was here ####
-POSTHOOK: query: delete from ice_orc where last_name in ('ln5a', 'ln6a',
'ln7a')
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@ice_orc
-#### A masked pattern was here ####
PREHOOK: query: select * from ice_orc
PREHOOK: type: QUERY
PREHOOK: Input: default@ice_orc
@@ -158,10 +54,10 @@ POSTHOOK: query: select * from ice_orc
POSTHOOK: type: QUERY
POSTHOOK: Input: default@ice_orc
#### A masked pattern was here ####
-fn1 ln1a
-fn2 ln2a
-fn3 ln3a
-fn4 ln4a
+fn1 ln1
+fn2 ln2
+fn3 ln3
+fn4 ln4
PREHOOK: query: describe formatted ice_orc
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_orc
@@ -184,21 +80,21 @@ Table Parameters:
bucketing_version 2
current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"}]}
current-snapshot-id #Masked#
- current-snapshot-summary
{\"deleted-data-files\":\"3\",\"deleted-records\":\"3\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"11\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"11\",\"total-delete-files\":\"7\",\"total-position-deletes\":\"7\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"}
+ current-snapshot-summary
{\"added-position-delete-files\":\"1\",\"added-delete-files\":\"1\",\"added-files-size\":\"#Masked#\",\"added-position-deletes\":\"3\",\"changed-partition-count\":\"1\",\"total-records\":\"7\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"1\",\"total-position-deletes\":\"3\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"}
current-snapshot-timestamp-ms #Masked#
format-version 2
iceberg.orc.files.only true
#### A masked pattern was here ####
- numFiles 11
- numRows 11
+ numFiles 1
+ numRows 4
parquet.compression zstd
#### A masked pattern was here ####
rawDataSize 0
serialization.format 1
- snapshot-count 15
+ snapshot-count 2
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
- totalSize #Masked#
+ totalSize #Masked#
#### A masked pattern was here ####
uuid #Masked#
write.delete.mode merge-on-read
@@ -212,11 +108,32 @@ InputFormat:
org.apache.iceberg.mr.hive.HiveIcebergInputFormat
OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
Compressed: No
Sort Columns: []
-PREHOOK: query: explain optimize table ice_orc rewrite data
+PREHOOK: query: explain alter table ice_orc COMPACT 'major' and wait order by
last_name desc
+PREHOOK: type: ALTERTABLE_COMPACT
+PREHOOK: Input: default@ice_orc
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: explain alter table ice_orc COMPACT 'major' and wait order by
last_name desc
+POSTHOOK: type: ALTERTABLE_COMPACT
+POSTHOOK: Input: default@ice_orc
+POSTHOOK: Output: default@ice_orc
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Compact
+ compaction type: major
+ table name: default.ice_orc
+ numberOfBuckets: 0
+ order by: order by last_name desc
+ table name: default.ice_orc
+ blocking: true
+
+PREHOOK: query: explain optimize table ice_orc rewrite data order by last_name
desc
PREHOOK: type: ALTERTABLE_COMPACT
PREHOOK: Input: default@ice_orc
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: explain optimize table ice_orc rewrite data
+POSTHOOK: query: explain optimize table ice_orc rewrite data order by
last_name desc
POSTHOOK: type: ALTERTABLE_COMPACT
POSTHOOK: Input: default@ice_orc
POSTHOOK: Output: default@ice_orc
@@ -229,14 +146,15 @@ STAGE PLANS:
compaction type: major
table name: default.ice_orc
numberOfBuckets: 0
+ order by: order by last_name desc
table name: default.ice_orc
blocking: true
-PREHOOK: query: optimize table ice_orc rewrite data
+PREHOOK: query: alter table ice_orc COMPACT 'major' and wait order by
last_name desc
PREHOOK: type: ALTERTABLE_COMPACT
PREHOOK: Input: default@ice_orc
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: optimize table ice_orc rewrite data
+POSTHOOK: query: alter table ice_orc COMPACT 'major' and wait order by
last_name desc
POSTHOOK: type: ALTERTABLE_COMPACT
POSTHOOK: Input: default@ice_orc
POSTHOOK: Output: default@ice_orc
@@ -248,10 +166,10 @@ POSTHOOK: query: select * from ice_orc
POSTHOOK: type: QUERY
POSTHOOK: Input: default@ice_orc
#### A masked pattern was here ####
-fn1 ln1a
-fn2 ln2a
-fn3 ln3a
-fn4 ln4a
+fn4 ln4
+fn3 ln3
+fn2 ln2
+fn1 ln1
PREHOOK: query: describe formatted ice_orc
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_orc
@@ -274,7 +192,7 @@ Table Parameters:
bucketing_version 2
current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"}]}
current-snapshot-id #Masked#
- current-snapshot-summary
{\"added-data-files\":\"1\",\"deleted-data-files\":\"11\",\"removed-position-delete-files\":\"7\",\"removed-delete-files\":\"7\",\"added-records\":\"4\",\"deleted-records\":\"11\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"removed-position-deletes\":\"7\",\"changed-partition-count\":\"1\",\"total-records\":\"4\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",
[...]
+ current-snapshot-summary
{\"added-data-files\":\"1\",\"deleted-data-files\":\"1\",\"removed-position-delete-files\":\"1\",\"removed-delete-files\":\"1\",\"added-records\":\"4\",\"deleted-records\":\"7\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"removed-position-deletes\":\"3\",\"changed-partition-count\":\"1\",\"total-records\":\"4\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"
[...]
current-snapshot-timestamp-ms #Masked#
format-version 2
iceberg.orc.files.only true
@@ -285,10 +203,10 @@ Table Parameters:
#### A masked pattern was here ####
rawDataSize 0
serialization.format 1
- snapshot-count 16
+ snapshot-count 3
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
- totalSize #Masked#
+ totalSize #Masked#
#### A masked pattern was here ####
uuid #Masked#
write.delete.mode merge-on-read
diff --git a/itests/src/test/resources/testconfiguration.properties
b/itests/src/test/resources/testconfiguration.properties
index 584a5fdba02..53665ee3e4d 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -422,6 +422,7 @@ iceberg.llap.query.files=\
iceberg.llap.query.compactor.files=\
iceberg_major_compaction_partition_evolution.q,\
iceberg_major_compaction_partition_evolution2.q,\
+ iceberg_major_compaction_partition_evolution_ordered.q,\
iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q,\
iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q,\
iceberg_major_compaction_partitioned.q,\
@@ -431,8 +432,8 @@ iceberg.llap.query.compactor.files=\
iceberg_major_compaction_single_partition_with_evolution.q,\
iceberg_major_compaction_single_partition_with_evolution2.q,\
iceberg_major_compaction_unpartitioned.q,\
- iceberg_major_compaction_unpartitioned_w_filter.q,\
- iceberg_optimize_table_unpartitioned.q
+ iceberg_major_compaction_unpartitioned_ordered.q,\
+ iceberg_major_compaction_unpartitioned_w_filter.q
iceberg.llap.only.query.files=\
iceberg_clustered.q,\
diff --git
a/parser/src/java/org/apache/hadoop/hive/ql/parse/AlterClauseParser.g
b/parser/src/java/org/apache/hadoop/hive/ql/parse/AlterClauseParser.g
index 2c95009529f..5a5b51673cd 100644
--- a/parser/src/java/org/apache/hadoop/hive/ql/parse/AlterClauseParser.g
+++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/AlterClauseParser.g
@@ -114,8 +114,8 @@ optimizeTableStatementSuffix
optimizeTblRewriteDataSuffix
@init { gParent.msgs.push("compaction request"); }
@after { gParent.msgs.pop(); }
- : KW_REWRITE KW_DATA whereClause?
- -> ^(TOK_ALTERTABLE_COMPACT Identifier["'MAJOR'"] TOK_BLOCKING
whereClause?)
+ : KW_REWRITE KW_DATA orderByClause? whereClause?
+ -> ^(TOK_ALTERTABLE_COMPACT Identifier["'MAJOR'"] TOK_BLOCKING
orderByClause? whereClause?)
;
alterStatementPartitionKeyType
diff --git
a/parser/src/test/org/apache/hadoop/hive/ql/parse/TestParseOptimizeTable.java
b/parser/src/test/org/apache/hadoop/hive/ql/parse/TestParseOptimizeTable.java
index 1f37f2252f8..744d603104c 100644
---
a/parser/src/test/org/apache/hadoop/hive/ql/parse/TestParseOptimizeTable.java
+++
b/parser/src/test/org/apache/hadoop/hive/ql/parse/TestParseOptimizeTable.java
@@ -24,32 +24,57 @@ import static org.hamcrest.core.Is.is;
public class TestParseOptimizeTable {
ParseDriver parseDriver = new ParseDriver();
- private static final String EXPECTED_WHERE_FILTER = "\n" +
- "nil\n" +
- " TOK_ALTERTABLE\n" +
- " TOK_TABNAME\n" +
- " tbl0\n" +
- " TOK_ALTERTABLE_COMPACT\n" +
- " 'MAJOR'\n" +
- " TOK_BLOCKING\n" +
- " TOK_WHERE\n" +
- " and\n" +
- " TOK_FUNCTION\n" +
- " in\n" +
- " TOK_TABLE_OR_COL\n" +
- " col01\n" +
- " 'A'\n" +
- " 'B'\n" +
- " <\n" +
- " TOK_TABLE_OR_COL\n" +
- " col02\n" +
- " '2024-09-17 00:00:00'\n" +
- " <EOF>\n";
+
@Test
public void testOptimizeTableWithWhere() throws Exception {
+ String EXPECTED_WHERE_FILTER = "\n" +
+ "nil\n" +
+ " TOK_ALTERTABLE\n" +
+ " TOK_TABNAME\n" +
+ " tbl0\n" +
+ " TOK_ALTERTABLE_COMPACT\n" +
+ " 'MAJOR'\n" +
+ " TOK_BLOCKING\n" +
+ " TOK_WHERE\n" +
+ " and\n" +
+ " TOK_FUNCTION\n" +
+ " in\n" +
+ " TOK_TABLE_OR_COL\n" +
+ " col01\n" +
+ " 'A'\n" +
+ " 'B'\n" +
+ " <\n" +
+ " TOK_TABLE_OR_COL\n" +
+ " col02\n" +
+ " '2024-09-17 00:00:00'\n" +
+ " <EOF>\n";
+
ASTNode tree = parseDriver.parse(
" optimize table tbl0 rewrite data where (col01 in ('A', 'B') and
col02 < '2024-09-17 00:00:00')", null).getTree();
assertThat(tree.dump(), is(EXPECTED_WHERE_FILTER));
}
+
+ @Test
+ public void testOptimizeTableWithOrderBy() throws Exception {
+
+ String EXPECTED_ORDER_BY = "\n" +
+ "nil\n" +
+ " TOK_ALTERTABLE\n" +
+ " TOK_TABNAME\n" +
+ " tbl0\n" +
+ " TOK_ALTERTABLE_COMPACT\n" +
+ " 'MAJOR'\n" +
+ " TOK_BLOCKING\n" +
+ " TOK_ORDERBY\n" +
+ " TOK_TABSORTCOLNAMEDESC\n" +
+ " TOK_NULLS_FIRST\n" +
+ " TOK_TABLE_OR_COL\n" +
+ " col01\n" +
+ " <EOF>\n";
+
+ ASTNode tree = parseDriver.parse(
+ " optimize table tbl0 rewrite data order by col01 desc",
null).getTree();
+ assertThat(tree.dump(), is(EXPECTED_ORDER_BY));
+ }
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorUtil.java
b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorUtil.java
index 6f953a63b9a..b97d67015c2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorUtil.java
@@ -341,6 +341,7 @@ public class CompactorUtil {
}
compactionRequest.setInitiatorVersion(ci.initiatorVersion);
compactionRequest.setPoolName(ci.poolName);
+ compactionRequest.setOrderByClause(ci.orderByClause);
LOG.info("Requesting compaction: " + compactionRequest);
CompactionResponse resp = txnHandler.compact(compactionRequest);
if (resp.isAccepted()) {