This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 4fb5e4f1093 HIVE-28342: Iceberg: Major QB Compaction filter support
(Dmitriy Fingerman, reviewed by Denys Kuzmenko, Indhumathi)
4fb5e4f1093 is described below
commit 4fb5e4f1093b010a4bc5292109cc23ee2e734b8f
Author: Dmitriy Fingerman <[email protected]>
AuthorDate: Thu Sep 12 10:58:21 2024 -0400
HIVE-28342: Iceberg: Major QB Compaction filter support (Dmitriy Fingerman,
reviewed by Denys Kuzmenko, Indhumathi)
Closes #5393
---
.../java/org/apache/hadoop/hive/ql/ErrorMsg.java | 2 +
.../iceberg/mr/hive/HiveIcebergStorageHandler.java | 79 +++--
.../apache/iceberg/mr/hive/IcebergTableUtil.java | 29 ++
.../compaction/IcebergMajorQueryCompactor.java | 46 +--
.../iceberg_major_compaction_partition_evolution.q | 2 +-
...iceberg_major_compaction_partition_evolution2.q | 4 +-
...ction_partition_evolution_w_dyn_spec_w_filter.q | 85 +++++
...ction_partition_evolution_w_id_spec_w_filter.q} | 53 +--
.../iceberg_major_compaction_partitioned.q | 4 +-
.../iceberg_major_compaction_schema_evolution.q | 2 +-
.../iceberg_major_compaction_single_partition.q | 4 +-
...or_compaction_single_partition_with_evolution.q | 2 +-
...r_compaction_single_partition_with_evolution2.q | 1 +
...berg_major_compaction_unpartitioned_w_filter.q} | 31 +-
...berg_major_compaction_partition_evolution.q.out | 4 +-
...erg_major_compaction_partition_evolution2.q.out | 8 +-
...n_partition_evolution_w_dyn_spec_w_filter.q.out | 387 +++++++++++++++++++++
...n_partition_evolution_w_id_spec_w_filter.q.out} | 227 +++++-------
.../iceberg_major_compaction_partitioned.q.out | 10 +-
...iceberg_major_compaction_schema_evolution.q.out | 4 +-
...iceberg_major_compaction_single_partition.q.out | 8 +-
...ompaction_single_partition_with_evolution.q.out | 8 +-
...mpaction_single_partition_with_evolution2.q.out | 7 +
..._major_compaction_unpartitioned_w_filter.q.out} | 151 +++-----
.../test/resources/testconfiguration.properties | 3 +
.../hadoop/hive/ql/parse/AlterClauseParser.g | 4 +-
.../storage/compact/AlterTableCompactAnalyzer.java | 30 +-
.../storage/compact/AlterTableCompactDesc.java | 11 +-
.../compact/AlterTableCompactOperation.java | 25 +-
.../concatenate/AlterTableConcatenateAnalyzer.java | 2 +-
.../hadoop/hive/ql/metadata/DummyPartition.java | 14 +
.../hive/ql/metadata/HiveStorageHandler.java | 36 +-
32 files changed, 890 insertions(+), 393 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
b/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
index 15174dcc500..12dfd1b8e0e 100644
--- a/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
+++ b/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
@@ -489,6 +489,8 @@ public enum ErrorMsg {
HiveConf.ConfVars.SPLIT_UPDATE.varname + "\"=\"true\""),
READ_ONLY_DATABASE(10436, "Database {0} is read-only", true),
UNEXPECTED_PARTITION_TRANSFORM_SPEC(10437, "Partition transforms are only
supported by Iceberg storage handler", true),
+ NONICEBERG_COMPACTION_WITH_FILTER_NOT_SUPPORTED(10440, "Compaction with
filter is not allowed on non-Iceberg table {0}.{1}", true),
+ ICEBERG_COMPACTION_WITH_PART_SPEC_AND_FILTER_NOT_SUPPORTED(10441,
"Compaction command with both partition spec and filter is not supported on
Iceberg table {0}.{1}", true),
//========================== 20000 range starts here
========================//
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
index ded1d194358..edb3a7a9e16 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
@@ -42,6 +42,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Function;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.collections4.ListUtils;
import org.apache.commons.lang3.SerializationUtils;
@@ -148,14 +149,11 @@ import org.apache.iceberg.FindFiles;
import org.apache.iceberg.GenericBlobMetadata;
import org.apache.iceberg.GenericStatisticsFile;
import org.apache.iceberg.ManifestFile;
-import org.apache.iceberg.MetadataTableType;
-import org.apache.iceberg.MetadataTableUtils;
import org.apache.iceberg.NullOrder;
import org.apache.iceberg.PartitionData;
import org.apache.iceberg.PartitionField;
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.PartitionSpecParser;
-import org.apache.iceberg.PartitionsTable;
import org.apache.iceberg.Schema;
import org.apache.iceberg.SchemaParser;
import org.apache.iceberg.SerializableTable;
@@ -205,7 +203,6 @@ import org.apache.iceberg.util.ByteBuffers;
import org.apache.iceberg.util.Pair;
import org.apache.iceberg.util.SerializationUtil;
import org.apache.iceberg.util.SnapshotUtil;
-import org.apache.iceberg.util.StructProjection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -391,9 +388,12 @@ public class HiveIcebergStorageHandler implements
HiveStoragePredicateHandler, H
ExprNodeDesc pushedPredicate = exprNodeDesc.clone();
List<ExprNodeDesc> subExprNodes = pushedPredicate.getChildren();
+ Set<String> skipList = Stream.of(VirtualColumn.FILE_PATH,
VirtualColumn.PARTITION_SPEC_ID,
+ VirtualColumn.PARTITION_HASH)
+ .map(VirtualColumn::getName).collect(Collectors.toSet());
+
if (subExprNodes.removeIf(nodeDesc -> nodeDesc.getCols() != null &&
- (nodeDesc.getCols().contains(VirtualColumn.FILE_PATH.getName()) ||
-
nodeDesc.getCols().contains(VirtualColumn.PARTITION_SPEC_ID.getName())))) {
+ nodeDesc.getCols().stream().anyMatch(skipList::contains))) {
if (subExprNodes.size() == 1) {
pushedPredicate = subExprNodes.get(0);
} else if (subExprNodes.isEmpty()) {
@@ -2067,32 +2067,55 @@ public class HiveIcebergStorageHandler implements
HiveStoragePredicateHandler, H
}
@Override
- public List<Partition>
getPartitionsByExpr(org.apache.hadoop.hive.ql.metadata.Table hmsTable,
ExprNodeDesc desc)
+ public List<Partition>
getPartitionsByExpr(org.apache.hadoop.hive.ql.metadata.Table hmsTable,
ExprNodeDesc filter)
throws SemanticException {
- Table icebergTable = IcebergTableUtil.getTable(conf, hmsTable.getTTable());
- PartitionSpec pSpec = icebergTable.spec();
- PartitionsTable partitionsTable = (PartitionsTable) MetadataTableUtils
- .createMetadataTableInstance(icebergTable,
MetadataTableType.PARTITIONS);
- SearchArgument sarg = ConvertAstToSearchArg.create(conf,
(ExprNodeGenericFuncDesc) desc);
- Expression expression =
HiveIcebergFilterFactory.generateFilterExpression(sarg);
- Set<PartitionData> partitionList = Sets.newHashSet();
- ResidualEvaluator resEval = ResidualEvaluator.of(pSpec, expression, false);
- try (CloseableIterable<FileScanTask> fileScanTasks =
partitionsTable.newScan().planFiles()) {
- fileScanTasks.forEach(task ->
-
partitionList.addAll(Sets.newHashSet(CloseableIterable.transform(task.asDataTask().rows(),
row -> {
- StructProjection data = row.get(IcebergTableUtil.PART_IDX,
StructProjection.class);
- return IcebergTableUtil.toPartitionData(data,
pSpec.partitionType());
- })).stream()
- .filter(partitionData ->
resEval.residualFor(partitionData).isEquivalentTo(Expressions.alwaysTrue()))
- .collect(Collectors.toSet())));
-
-
- return partitionList.stream()
- .map(partitionData -> new DummyPartition(hmsTable,
pSpec.partitionToPath(partitionData)))
- .collect(Collectors.toList());
+ return getPartitionsByExpr(hmsTable, filter, true);
+ }
+
+ @Override
+ public List<Partition>
getPartitionsByExpr(org.apache.hadoop.hive.ql.metadata.Table hmsTable,
+ ExprNodeDesc filter, boolean latestSpecOnly) throws SemanticException {
+ SearchArgument sarg = ConvertAstToSearchArg.create(conf,
(ExprNodeGenericFuncDesc) filter);
+ Expression exp = HiveIcebergFilterFactory.generateFilterExpression(sarg);
+ Table table = IcebergTableUtil.getTable(conf, hmsTable.getTTable());
+ int tableSpecId = table.spec().specId();
+ Set<Partition> partitions = Sets.newHashSet();
+
+ TableScan scan =
table.newScan().filter(exp).caseSensitive(false).includeColumnStats().ignoreResiduals();
+
+ try (CloseableIterable<FileScanTask> tasks = scan.planFiles()) {
+ tasks.forEach(task -> {
+ DataFile file = task.file();
+ PartitionSpec spec = task.spec();
+ if ((latestSpecOnly && file.specId() == tableSpecId) ||
(!latestSpecOnly && file.specId() != tableSpecId)) {
+ PartitionData partitionData =
IcebergTableUtil.toPartitionData(task.partition(), spec.partitionType());
+ String partName = spec.partitionToPath(partitionData);
+ Map<String, String> partSpecMap = Maps.newLinkedHashMap();
+ Warehouse.makeSpecFromName(partSpecMap, new Path(partName), null);
+ DummyPartition partition = new DummyPartition(hmsTable, partName,
partSpecMap);
+ partitions.add(partition);
+ }
+ });
} catch (IOException e) {
throw new SemanticException(String.format("Error while fetching the
partitions due to: %s", e));
}
+ return ImmutableList.copyOf(partitions);
+ }
+
+ @Override
+ public boolean
hasDataMatchingFilterExpr(org.apache.hadoop.hive.ql.metadata.Table hmsTable,
ExprNodeDesc filter) {
+ SearchArgument sarg = ConvertAstToSearchArg.create(conf,
(ExprNodeGenericFuncDesc) filter);
+ Expression exp = HiveIcebergFilterFactory.generateFilterExpression(sarg);
+ Table table = IcebergTableUtil.getTable(conf, hmsTable.getTTable());
+ TableScan scan =
table.newScan().filter(exp).caseSensitive(false).includeColumnStats().ignoreResiduals();
+ boolean result = false;
+
+ try (CloseableIterable<FileScanTask> tasks = scan.planFiles()) {
+ result = tasks.iterator().hasNext();
+ } catch (IOException ioe) {
+ LOG.warn("Failed to close task iterable", ioe);
+ }
+ return result;
}
@Override
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java
index 627e6f4b101..792265eb184 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java
@@ -72,6 +72,7 @@ import org.apache.iceberg.hive.HiveSchemaUtil;
import org.apache.iceberg.io.CloseableIterable;
import org.apache.iceberg.mr.Catalogs;
import org.apache.iceberg.mr.InputFormatConfig;
+import org.apache.iceberg.relocated.com.google.common.collect.FluentIterable;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
import org.apache.iceberg.types.Conversions;
@@ -483,6 +484,14 @@ public class IcebergTableUtil {
.stream()).distinct().collect(Collectors.toList());
}
+ /**
+ * Returns a Map of PartitionData as the keys and partition spec ids as the
values
+ * @param icebergTable Iceberg table
+ * @param partSpecMap Partition Spec used as the criteria for filtering
+ * @param allowPartialSpec When true, must return partitions which match
partSpecMap exactly, otherwise partially
+ * @param latestSpecOnly When true, returns partitions with the latest
partition spec only, otherwise with any specs
+ * @return Map of PartitionData and partition spec found based on the
specified constraints
+ */
public static Map<PartitionData, Integer> getPartitionInfo(Table
icebergTable, Map<String, String> partSpecMap,
boolean allowPartialSpec, boolean latestSpecOnly) throws
SemanticException, IOException {
Expression expression =
IcebergTableUtil.generateExpressionFromPartitionSpec(icebergTable, partSpecMap);
@@ -511,6 +520,26 @@ public class IcebergTableUtil {
return result;
}
+ public static long getPartitionHash(Table icebergTable, String
partitionPath) throws IOException {
+ PartitionsTable partitionsTable = (PartitionsTable) MetadataTableUtils
+ .createMetadataTableInstance(icebergTable,
MetadataTableType.PARTITIONS);
+ try (CloseableIterable<FileScanTask> fileScanTasks =
partitionsTable.newScan().planFiles()) {
+ return FluentIterable.from(fileScanTasks)
+ .transformAndConcat(task -> task.asDataTask().rows())
+ .transform(row -> {
+ StructProjection data = row.get(IcebergTableUtil.PART_IDX,
StructProjection.class);
+ PartitionSpec spec =
icebergTable.specs().get(row.get(IcebergTableUtil.SPEC_IDX, Integer.class));
+ PartitionData partitionData =
IcebergTableUtil.toPartitionData(data,
+ Partitioning.partitionType(icebergTable),
spec.partitionType());
+ String path = spec.partitionToPath(partitionData);
+ return Maps.immutableEntry(path, data);
+ })
+ .filter(e -> e.getKey().equals(partitionPath))
+ .transform(e -> IcebergAcidUtil.computeHash(e.getValue()))
+ .get(0);
+ }
+ }
+
public static List<String> getPartitionNames(Table icebergTable, Map<String,
String> partitionSpec,
boolean latestSpecOnly) throws SemanticException {
try {
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergMajorQueryCompactor.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergMajorQueryCompactor.java
index 906854070d1..d200eafd1b9 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergMajorQueryCompactor.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergMajorQueryCompactor.java
@@ -21,16 +21,11 @@ package org.apache.iceberg.mr.hive.compaction;
import java.io.IOException;
import java.util.LinkedHashMap;
-import java.util.List;
import java.util.Map;
-import java.util.Optional;
-import java.util.stream.Collectors;
-import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.Warehouse;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.Context.RewritePolicy;
import org.apache.hadoop.hive.ql.DriverUtils;
import org.apache.hadoop.hive.ql.ErrorMsg;
@@ -41,10 +36,7 @@ import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.txn.compactor.CompactorContext;
import org.apache.hadoop.hive.ql.txn.compactor.QueryCompactor;
-import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.apache.hive.iceberg.org.apache.orc.storage.common.TableName;
-import org.apache.iceberg.PartitionData;
import org.apache.iceberg.Table;
import org.apache.iceberg.mr.hive.IcebergTableUtil;
import org.slf4j.Logger;
@@ -87,42 +79,16 @@ public class IcebergMajorQueryCompactor extends
QueryCompactor {
throw new HiveException(ErrorMsg.COMPACTION_NO_PARTITION);
}
} else {
- Map<String, String> partSpecMap = new LinkedHashMap<>();
- Warehouse.makeSpecFromName(partSpecMap, new Path(partSpec), null);
- Map<PartitionData, Integer> partitionInfo = IcebergTableUtil
- .getPartitionInfo(icebergTable, partSpecMap, false, false);
- Optional<Integer> specId = partitionInfo.values().stream().findFirst();
-
- if (!specId.isPresent()) {
- throw new HiveException(ErrorMsg.INVALID_PARTITION_SPEC);
- }
-
- if (partitionInfo.size() > 1) {
- throw new HiveException(ErrorMsg.TOO_MANY_COMPACTION_PARTITIONS);
- }
-
+ long partitionHash = IcebergTableUtil.getPartitionHash(icebergTable,
partSpec);
HiveConf.setVar(conf, ConfVars.REWRITE_POLICY,
RewritePolicy.PARTITION.name());
conf.set(IcebergCompactionService.PARTITION_PATH, new
Path(partSpec).toString());
- List<FieldSchema> partitionKeys =
IcebergTableUtil.getPartitionKeys(icebergTable, specId.get());
- List<String> partValues = partitionKeys.stream().map(
- fs -> String.join("=", HiveUtils.unparseIdentifier(fs.getName()),
-
TypeInfoUtils.convertStringToLiteralForSQL(partSpecMap.get(fs.getName()),
- ((PrimitiveTypeInfo)
TypeInfoUtils.getTypeInfoFromTypeString(fs.getType())).getPrimitiveCategory())
- )
- ).collect(Collectors.toList());
-
- String queryFields = table.getCols().stream()
- .map(FieldSchema::getName)
- .filter(col -> !partSpecMap.containsKey(col))
- .collect(Collectors.joining(","));
+ Map<String, String> partSpecMap = new LinkedHashMap<>();
+ Warehouse.makeSpecFromName(partSpecMap, new Path(partSpec), null);
- compactionQuery = String.format("insert overwrite table %1$s
partition(%2$s) " +
- "select %4$s from %1$s where %3$s and %6$s = %5$d",
- compactTableName,
- StringUtils.join(partValues, ","),
- StringUtils.join(partValues, " and "),
- queryFields, specId.get(),
VirtualColumn.PARTITION_SPEC_ID.getName());
+ compactionQuery = String.format("insert overwrite table %1$s select *
from %1$s where %2$s=%3$d " +
+ "and %4$s is not null", compactTableName,
VirtualColumn.PARTITION_HASH.getName(), partitionHash,
+ VirtualColumn.FILE_PATH.getName());
}
SessionState sessionState = setupQueryCompactionSession(conf,
context.getCompactionInfo(), tblProperties);
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution.q
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution.q
index 7d046aea4e9..3ee23487568 100644
---
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution.q
+++
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution.q
@@ -62,4 +62,4 @@ alter table ice_orc COMPACT 'major' and wait;
select * from ice_orc;
describe formatted ice_orc;
-show compactions;
\ No newline at end of file
+show compactions order by 'partition';
\ No newline at end of file
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution2.q
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution2.q
index 3fa6f91f90b..5ea769881cd 100644
---
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution2.q
+++
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution2.q
@@ -45,11 +45,11 @@ delete from ice_orc where last_name in ('ln7', 'ln8');
select * from ice_orc;
describe formatted ice_orc;
-show compactions;
+show compactions order by 'partition';
alter table ice_orc COMPACT 'major' and wait;
select * from ice_orc;
describe formatted ice_orc;
-show compactions;
+show compactions order by 'partition';
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q
new file mode 100644
index 00000000000..e63c99a3953
--- /dev/null
+++
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q
@@ -0,0 +1,85 @@
+-- SORT_QUERY_RESULTS
+-- Mask neededVirtualColumns due to non-strict order
+--! qt:replace:/(\s+neededVirtualColumns:\s)(.*)/$1#Masked#/
+-- Mask random uuid
+--! qt:replace:/(\s+uuid\s+)\S+(\s*)/$1#Masked#$2/
+-- Mask a random snapshot id
+--! qt:replace:/(\s+current-snapshot-id\s+)\S+(\s*)/$1#Masked#/
+-- Mask added file size
+--! qt:replace:/(\S\"added-files-size\\\":\\\")(\d+)(\\\")/$1#Masked#$3/
+-- Mask total file size
+--! qt:replace:/(\S\"total-files-size\\\":\\\")(\d+)(\\\")/$1#Masked#$3/
+-- Mask current-snapshot-timestamp-ms
+--! qt:replace:/(\s+current-snapshot-timestamp-ms\s+)\S+(\s*)/$1#Masked#$2/
+--!
qt:replace:/(MAJOR\s+succeeded\s+)[a-zA-Z0-9\-\.\s+]+(\s+manual)/$1#Masked#$2/
+-- Mask compaction id as they will be allocated in parallel threads
+--! qt:replace:/^[0-9]/#Masked#/
+-- Mask removed file size
+--! qt:replace:/(\S\"removed-files-size\\\":\\\")(\d+)(\\\")/$1#Masked#$3/
+
+set hive.llap.io.enabled=true;
+set hive.vectorized.execution.enabled=true;
+set hive.optimize.shared.work.merge.ts.schema=true;
+
+create table ice_orc (
+ event_id int,
+ event_time timestamp with local time zone,
+ event_src string
+)
+partitioned by spec(truncate(3, event_src))
+stored by iceberg stored as orc;
+
+insert into ice_orc values
+ (1, cast('2023-07-20 00:00:00' as timestamp with local time zone),
'AAA_1'),
+ (2, cast('2023-07-21 00:00:00' as timestamp with local time zone),
'AAA_2');
+insert into ice_orc values
+ (3, cast('2023-07-23 00:00:00' as timestamp with local time zone),
'AAA_1'),
+ (4, cast('2023-07-24 00:00:00' as timestamp with local time zone),
'AAA_2');
+insert into ice_orc values
+ (5, cast('2023-08-04 00:00:00' as timestamp with local time zone),
'BBB_1'),
+ (6, cast('2023-08-05 00:00:00' as timestamp with local time zone),
'BBB_2');
+insert into ice_orc values
+ (7, cast('2023-08-06 00:00:00' as timestamp with local time zone),
'BBB_1'),
+ (8, cast('2023-08-27 00:00:00' as timestamp with local time zone),
'BBB_2');
+
+alter table ice_orc set partition spec(truncate(3, event_src),
month(event_time));
+
+insert into ice_orc values
+ (9, cast('2024-07-20 00:00:00' as timestamp with local time zone),
'AAA_1'),
+ (10, cast('2024-07-21 00:00:00' as timestamp with local time zone),
'AAA_2');
+insert into ice_orc values
+ (11, cast('2024-08-22 00:00:00' as timestamp with local time zone),
'AAA_1'),
+ (12, cast('2024-08-23 00:00:00' as timestamp with local time zone),
'AAA_2');
+insert into ice_orc values
+ (13, cast('2024-08-24 00:00:00' as timestamp with local time zone),
'AAA_1'),
+ (14, cast('2024-08-25 00:00:00' as timestamp with local time zone),
'AAA_2');
+insert into ice_orc values
+ (15, cast('2024-09-24 00:00:00' as timestamp with local time zone),
'AAA_1'),
+ (16, cast('2024-09-25 00:00:00' as timestamp with local time zone),
'AAA_2');
+insert into ice_orc values
+ (17, cast('2024-07-01 00:00:00' as timestamp with local time zone),
'BBB_1'),
+ (18, cast('2024-07-02 00:00:00' as timestamp with local time zone),
'BBB_2');
+insert into ice_orc values
+ (19, cast('2024-07-03 00:00:00' as timestamp with local time zone),
'BBB_1'),
+ (20, cast('2024-07-04 00:00:00' as timestamp with local time zone),
'BBB_2');
+insert into ice_orc values
+ (21, cast('2024-08-03 00:00:00' as timestamp with local time zone),
'BBB_1'),
+ (22, cast('2024-08-04 00:00:00' as timestamp with local time zone),
'BBB_2');
+insert into ice_orc values
+ (23, cast('2024-08-05 00:00:00' as timestamp with local time zone),
'BBB_1'),
+ (24, cast('2024-08-06 00:00:00' as timestamp with local time zone),
'BBB_2');
+insert into ice_orc values
+ (25, cast('2024-09-05 00:00:00' as timestamp with local time zone),
'BBB_1'),
+ (26, cast('2024-09-06 00:00:00' as timestamp with local time zone),
'BBB_2');
+
+select * from ice_orc;
+describe formatted ice_orc;
+
+alter table ice_orc COMPACT 'major' and wait
+where (event_src in ('BBB_1', 'BBB_2') and event_time < '2024-09-01 00:00:00')
or
+ (event_src in ('AAA_1', 'AAA_2') and event_time > '2024-08-01 00:00:00');
+
+select * from ice_orc;
+describe formatted ice_orc;
+
+show compactions order by 'partition';
\ No newline at end of file
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution.q
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q
similarity index 53%
copy from
iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution.q
copy to
iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q
index 7d046aea4e9..84e3eab0f7b 100644
---
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution.q
+++
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q
@@ -31,35 +31,44 @@ partitioned by (company_id bigint)
stored by iceberg stored as orc
tblproperties ('format-version'='2');
-insert into ice_orc VALUES ('fn1','ln1', 1, 10, 100);
-insert into ice_orc VALUES ('fn2','ln2', 1, 10, 100);
-insert into ice_orc VALUES ('fn3','ln3', 1, 11, 100);
+insert into ice_orc VALUES
+ ('fn1','ln1', 1, 10, 100),
+ ('fn2','ln2', 1, 10, 100);
+insert into ice_orc VALUES
+ ('fn3','ln3', 2, 11, 100),
+ ('fn4','ln4', 2, 11, 100);
+insert into ice_orc VALUES
+ ('fn5','ln5', 3, 12, 100),
+ ('fn6','ln6', 3, 12, 100);
+insert into ice_orc VALUES
+ ('fn7','ln7', 4, 13, 100),
+ ('fn8','ln8', 4, 13, 100);
+
alter table ice_orc set partition spec(company_id, dept_id);
-insert into ice_orc VALUES ('fn4','ln4', 1, 11, 100);
-insert into ice_orc VALUES ('fn5','ln5', 2, 20, 100);
-insert into ice_orc VALUES ('fn6','ln6', 2, 20, 100);
-alter table ice_orc set partition spec(company_id, dept_id, team_id);
-insert into ice_orc VALUES ('fn7','ln7', 2, 21, 100);
-insert into ice_orc VALUES ('fn8','ln8', 2, 21, 100);
-update ice_orc set last_name = 'ln1a' where first_name='fn1';
-update ice_orc set last_name = 'ln2a' where first_name='fn2';
-update ice_orc set last_name = 'ln3a' where first_name='fn3';
-update ice_orc set last_name = 'ln4a' where first_name='fn4';
-alter table ice_orc set partition spec(company_id, dept_id);
-update ice_orc set last_name = 'ln5a' where first_name='fn5';
-update ice_orc set last_name = 'ln6a' where first_name='fn6';
-update ice_orc set last_name = 'ln7a' where first_name='fn7';
-update ice_orc set last_name = 'ln8a' where first_name='fn8';
+insert into ice_orc VALUES
+ ('fn9', 'ln9', 1, 10, 100),
+ ('fn10','ln10', 1, 10, 100);
+insert into ice_orc VALUES
+ ('fn11','ln11', 2, 11, 100),
+ ('fn12','ln12', 2, 11, 100);
+insert into ice_orc VALUES
+ ('fn13','ln13', 3, 12, 100),
+ ('fn14','ln14', 3, 12, 100);
+insert into ice_orc VALUES
+ ('fn15','ln15', 4, 13, 100),
+ ('fn16','ln16', 4, 13, 100);
-delete from ice_orc where last_name in ('ln1a', 'ln8a');
+delete from ice_orc where last_name in ('ln1', 'ln9');
+delete from ice_orc where last_name in ('ln3', 'ln11');
+delete from ice_orc where last_name in ('ln5', 'ln13');
select * from ice_orc;
describe formatted ice_orc;
-explain alter table ice_orc COMPACT 'major' and wait;
-alter table ice_orc COMPACT 'major' and wait;
+explain alter table ice_orc COMPACT 'major' and wait where team_id=10 or
first_name in ('fn3', 'fn11') or last_name in ('ln7', 'ln15');
+alter table ice_orc COMPACT 'major' and wait where team_id=10 or first_name in
('fn3', 'fn11') or last_name in ('ln7', 'ln15');
select * from ice_orc;
describe formatted ice_orc;
-show compactions;
\ No newline at end of file
+show compactions order by 'partition';
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partitioned.q
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partitioned.q
index cc0d2fa23b9..256dc516186 100644
---
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partitioned.q
+++
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partitioned.q
@@ -55,7 +55,7 @@ alter table ice_orc COMPACT 'major' and wait;
select * from ice_orc;
describe formatted ice_orc;
-show compactions;
+show compactions order by 'partition';
-- Starting second set of inserts/updates/deletes and calling compaction at
the end
-- to check that subsequent compaction works
@@ -88,4 +88,4 @@ alter table ice_orc COMPACT 'major' and wait;
select * from ice_orc;
describe formatted ice_orc;
-show compactions;
+show compactions order by 'partition';
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_schema_evolution.q
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_schema_evolution.q
index 8501e694de0..2e57b36c57d 100644
---
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_schema_evolution.q
+++
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_schema_evolution.q
@@ -63,4 +63,4 @@ alter table ice_orc COMPACT 'major' and wait;
select * from ice_orc;
describe formatted ice_orc;
-show compactions;
\ No newline at end of file
+show compactions order by 'partition';
\ No newline at end of file
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_single_partition.q
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_single_partition.q
index 35ae60459b5..55d8ed4421e 100644
---
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_single_partition.q
+++
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_single_partition.q
@@ -59,11 +59,11 @@ alter table ice_orc_wo_evo PARTITION (dept_id=1,
city='London', registration_dat
select * from ice_orc_wo_evo;
describe formatted ice_orc_wo_evo;
-show compactions;
+show compactions order by 'partition';
explain alter table ice_orc_wo_evo PARTITION (dept_id=2, city='Paris',
registration_date='2024-02-16') COMPACT 'major' and wait;
alter table ice_orc_wo_evo PARTITION (dept_id=2, city='Paris',
registration_date='2024-02-16') COMPACT 'major' and wait;
select * from ice_orc_wo_evo;
describe formatted ice_orc_wo_evo;
-show compactions;
+show compactions order by 'partition';
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_single_partition_with_evolution.q
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_single_partition_with_evolution.q
index c724b249be6..ef8f37ada53 100644
---
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_single_partition_with_evolution.q
+++
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_single_partition_with_evolution.q
@@ -93,4 +93,4 @@ alter table ice_orc PARTITION (dept_id=2, city='Paris')
COMPACT 'major' and wait
select * from ice_orc;
describe formatted ice_orc;
-show compactions;
+show compactions order by 'partition';
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_single_partition_with_evolution2.q
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_single_partition_with_evolution2.q
index fa0de6ae6c4..5f5e0333bfd 100644
---
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_single_partition_with_evolution2.q
+++
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_single_partition_with_evolution2.q
@@ -70,3 +70,4 @@ alter table ice_orc partition(b=1) compact 'major' and wait;
select * from ice_orc;
describe formatted ice_orc;
+show compactions order by 'partition';
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution.q
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_unpartitioned_w_filter.q
similarity index 63%
copy from
iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution.q
copy to
iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_unpartitioned_w_filter.q
index 7d046aea4e9..f4328a9f8f6 100644
---
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution.q
+++
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_unpartitioned_w_filter.q
@@ -11,6 +11,7 @@
--! qt:replace:/(\S\"total-files-size\\\":\\\")(\d+)(\\\")/$1#Masked#$3/
-- Mask current-snapshot-timestamp-ms
--! qt:replace:/(\s+current-snapshot-timestamp-ms\s+)\S+(\s*)/$1#Masked#$2/
+-- Mask the enqueue time which is based on current time
--!
qt:replace:/(MAJOR\s+succeeded\s+)[a-zA-Z0-9\-\.\s+]+(\s+manual)/$1#Masked#$2/
-- Mask compaction id as they will be allocated in parallel threads
--! qt:replace:/^[0-9]/#Masked#/
@@ -23,42 +24,34 @@ set hive.optimize.shared.work.merge.ts.schema=true;
create table ice_orc (
first_name string,
- last_name string,
- dept_id bigint,
- team_id bigint
+ last_name string
)
-partitioned by (company_id bigint)
stored by iceberg stored as orc
tblproperties ('format-version'='2');
-insert into ice_orc VALUES ('fn1','ln1', 1, 10, 100);
-insert into ice_orc VALUES ('fn2','ln2', 1, 10, 100);
-insert into ice_orc VALUES ('fn3','ln3', 1, 11, 100);
-alter table ice_orc set partition spec(company_id, dept_id);
-insert into ice_orc VALUES ('fn4','ln4', 1, 11, 100);
-insert into ice_orc VALUES ('fn5','ln5', 2, 20, 100);
-insert into ice_orc VALUES ('fn6','ln6', 2, 20, 100);
-alter table ice_orc set partition spec(company_id, dept_id, team_id);
-insert into ice_orc VALUES ('fn7','ln7', 2, 21, 100);
-insert into ice_orc VALUES ('fn8','ln8', 2, 21, 100);
+insert into ice_orc VALUES ('fn1','ln1');
+insert into ice_orc VALUES ('fn2','ln2');
+insert into ice_orc VALUES ('fn3','ln3');
+insert into ice_orc VALUES ('fn4','ln4');
+insert into ice_orc VALUES ('fn5','ln5');
+insert into ice_orc VALUES ('fn6','ln6');
+insert into ice_orc VALUES ('fn7','ln7');
update ice_orc set last_name = 'ln1a' where first_name='fn1';
update ice_orc set last_name = 'ln2a' where first_name='fn2';
update ice_orc set last_name = 'ln3a' where first_name='fn3';
update ice_orc set last_name = 'ln4a' where first_name='fn4';
-alter table ice_orc set partition spec(company_id, dept_id);
update ice_orc set last_name = 'ln5a' where first_name='fn5';
update ice_orc set last_name = 'ln6a' where first_name='fn6';
update ice_orc set last_name = 'ln7a' where first_name='fn7';
-update ice_orc set last_name = 'ln8a' where first_name='fn8';
-delete from ice_orc where last_name in ('ln1a', 'ln8a');
+delete from ice_orc where last_name in ('ln5a', 'ln6a', 'ln7a');
select * from ice_orc;
describe formatted ice_orc;
-explain alter table ice_orc COMPACT 'major' and wait;
-alter table ice_orc COMPACT 'major' and wait;
+explain alter table ice_orc COMPACT 'major' and wait where last_name in
('ln1a', 'ln2a');
+alter table ice_orc COMPACT 'major' and wait where last_name in ('ln1a',
'ln2a');
select * from ice_orc;
describe formatted ice_orc;
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out
index d9dee1d9902..36591e146c2 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out
@@ -369,9 +369,9 @@ InputFormat:
org.apache.iceberg.mr.hive.HiveIcebergInputFormat
OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
Compressed: No
Sort Columns: []
-PREHOOK: query: show compactions
+PREHOOK: query: show compactions order by 'partition'
PREHOOK: type: SHOW COMPACTIONS
-POSTHOOK: query: show compactions
+POSTHOOK: query: show compactions order by 'partition'
POSTHOOK: type: SHOW COMPACTIONS
CompactionId Database Table Partition Type State Worker
host Worker Enqueue Time Start Time Duration(ms) HadoopJobId
Error message Initiator host Initiator Pool name TxnId Next
TxnId Commit Time Highest WriteId
#Masked# default ice_orc company_id=100/dept_id=1 MAJOR
succeeded #Masked# manual default 0 0 0 ---
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out
index 06611f2f27a..01c93561ced 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out
@@ -173,9 +173,9 @@ InputFormat:
org.apache.iceberg.mr.hive.HiveIcebergInputFormat
OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
Compressed: No
Sort Columns: []
-PREHOOK: query: show compactions
+PREHOOK: query: show compactions order by 'partition'
PREHOOK: type: SHOW COMPACTIONS
-POSTHOOK: query: show compactions
+POSTHOOK: query: show compactions order by 'partition'
POSTHOOK: type: SHOW COMPACTIONS
CompactionId Database Table Partition Type State Worker
host Worker Enqueue Time Start Time Duration(ms) HadoopJobId
Error message Initiator host Initiator Pool name TxnId Next
TxnId Commit Time Highest WriteId
PREHOOK: query: alter table ice_orc COMPACT 'major' and wait
@@ -254,9 +254,9 @@ InputFormat:
org.apache.iceberg.mr.hive.HiveIcebergInputFormat
OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
Compressed: No
Sort Columns: []
-PREHOOK: query: show compactions
+PREHOOK: query: show compactions order by 'partition'
PREHOOK: type: SHOW COMPACTIONS
-POSTHOOK: query: show compactions
+POSTHOOK: query: show compactions order by 'partition'
POSTHOOK: type: SHOW COMPACTIONS
CompactionId Database Table Partition Type State Worker
host Worker Enqueue Time Start Time Duration(ms) HadoopJobId
Error message Initiator host Initiator Pool name TxnId Next
TxnId Commit Time Highest WriteId
#Masked# default ice_orc dept_id=2 MAJOR succeeded
#Masked# manual default 0 0 0 ---
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out
new file mode 100644
index 00000000000..88690baa04c
--- /dev/null
+++
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out
@@ -0,0 +1,387 @@
+PREHOOK: query: create table ice_orc (
+ event_id int,
+ event_time timestamp with local time zone,
+ event_src string
+)
+partitioned by spec(truncate(3, event_src))
+stored by iceberg stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: create table ice_orc (
+ event_id int,
+ event_time timestamp with local time zone,
+ event_src string
+)
+partitioned by spec(truncate(3, event_src))
+stored by iceberg stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc values
+ (1, cast('2023-07-20 00:00:00' as timestamp with local time zone),
'AAA_1'),
+ (2, cast('2023-07-21 00:00:00' as timestamp with local time zone), 'AAA_2')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc values
+ (1, cast('2023-07-20 00:00:00' as timestamp with local time zone),
'AAA_1'),
+ (2, cast('2023-07-21 00:00:00' as timestamp with local time zone), 'AAA_2')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc values
+ (3, cast('2023-07-23 00:00:00' as timestamp with local time zone),
'AAA_1'),
+ (4, cast('2023-07-24 00:00:00' as timestamp with local time zone), 'AAA_2')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc values
+ (3, cast('2023-07-23 00:00:00' as timestamp with local time zone),
'AAA_1'),
+ (4, cast('2023-07-24 00:00:00' as timestamp with local time zone), 'AAA_2')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc values
+ (5, cast('2023-08-04 00:00:00' as timestamp with local time zone),
'BBB_1'),
+ (6, cast('2023-08-05 00:00:00' as timestamp with local time zone), 'BBB_2')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc values
+ (5, cast('2023-08-04 00:00:00' as timestamp with local time zone),
'BBB_1'),
+ (6, cast('2023-08-05 00:00:00' as timestamp with local time zone), 'BBB_2')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc values
+ (7, cast('2023-08-06 00:00:00' as timestamp with local time zone),
'BBB_1'),
+ (8, cast('2023-08-27 00:00:00' as timestamp with local time zone), 'BBB_2')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc values
+ (7, cast('2023-08-06 00:00:00' as timestamp with local time zone),
'BBB_1'),
+ (8, cast('2023-08-27 00:00:00' as timestamp with local time zone), 'BBB_2')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: alter table ice_orc set partition spec(truncate(3, event_src),
month(event_time))
+PREHOOK: type: ALTERTABLE_SETPARTSPEC
+PREHOOK: Input: default@ice_orc
+POSTHOOK: query: alter table ice_orc set partition spec(truncate(3,
event_src), month(event_time))
+POSTHOOK: type: ALTERTABLE_SETPARTSPEC
+POSTHOOK: Input: default@ice_orc
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc values
+ (9, cast('2024-07-20 00:00:00' as timestamp with local time zone),
'AAA_1'),
+ (10, cast('2024-07-21 00:00:00' as timestamp with local time zone),
'AAA_2')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc values
+ (9, cast('2024-07-20 00:00:00' as timestamp with local time zone),
'AAA_1'),
+ (10, cast('2024-07-21 00:00:00' as timestamp with local time zone),
'AAA_2')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc values
+ (11, cast('2024-08-22 00:00:00' as timestamp with local time zone),
'AAA_1'),
+ (12, cast('2024-08-23 00:00:00' as timestamp with local time zone),
'AAA_2')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc values
+ (11, cast('2024-08-22 00:00:00' as timestamp with local time zone),
'AAA_1'),
+ (12, cast('2024-08-23 00:00:00' as timestamp with local time zone),
'AAA_2')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc values
+ (13, cast('2024-08-24 00:00:00' as timestamp with local time zone),
'AAA_1'),
+ (14, cast('2024-08-25 00:00:00' as timestamp with local time zone),
'AAA_2')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc values
+ (13, cast('2024-08-24 00:00:00' as timestamp with local time zone),
'AAA_1'),
+ (14, cast('2024-08-25 00:00:00' as timestamp with local time zone),
'AAA_2')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc values
+ (15, cast('2024-09-24 00:00:00' as timestamp with local time zone),
'AAA_1'),
+ (16, cast('2024-09-25 00:00:00' as timestamp with local time zone),
'AAA_2')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc values
+ (15, cast('2024-09-24 00:00:00' as timestamp with local time zone),
'AAA_1'),
+ (16, cast('2024-09-25 00:00:00' as timestamp with local time zone),
'AAA_2')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc values
+ (17, cast('2024-07-01 00:00:00' as timestamp with local time zone),
'BBB_1'),
+ (18, cast('2024-07-02 00:00:00' as timestamp with local time zone),
'BBB_2')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc values
+ (17, cast('2024-07-01 00:00:00' as timestamp with local time zone),
'BBB_1'),
+ (18, cast('2024-07-02 00:00:00' as timestamp with local time zone),
'BBB_2')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc values
+ (19, cast('2024-07-03 00:00:00' as timestamp with local time zone),
'BBB_1'),
+ (20, cast('2024-07-04 00:00:00' as timestamp with local time zone),
'BBB_2')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc values
+ (19, cast('2024-07-03 00:00:00' as timestamp with local time zone),
'BBB_1'),
+ (20, cast('2024-07-04 00:00:00' as timestamp with local time zone),
'BBB_2')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc values
+ (21, cast('2024-08-03 00:00:00' as timestamp with local time zone),
'BBB_1'),
+ (22, cast('2024-08-04 00:00:00' as timestamp with local time zone),
'BBB_2')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc values
+ (21, cast('2024-08-03 00:00:00' as timestamp with local time zone),
'BBB_1'),
+ (22, cast('2024-08-04 00:00:00' as timestamp with local time zone),
'BBB_2')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc values
+ (23, cast('2024-08-05 00:00:00' as timestamp with local time zone),
'BBB_1'),
+ (24, cast('2024-08-06 00:00:00' as timestamp with local time zone),
'BBB_2')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc values
+ (23, cast('2024-08-05 00:00:00' as timestamp with local time zone),
'BBB_1'),
+ (24, cast('2024-08-06 00:00:00' as timestamp with local time zone),
'BBB_2')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: insert into ice_orc values
+ (25, cast('2024-09-05 00:00:00' as timestamp with local time zone),
'BBB_1'),
+ (26, cast('2024-09-06 00:00:00' as timestamp with local time zone),
'BBB_2')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: insert into ice_orc values
+ (25, cast('2024-09-05 00:00:00' as timestamp with local time zone),
'BBB_1'),
+ (26, cast('2024-09-06 00:00:00' as timestamp with local time zone),
'BBB_2')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: select * from ice_orc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_orc
+#### A masked pattern was here ####
+POSTHOOK: query: select * from ice_orc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_orc
+#### A masked pattern was here ####
+#Masked# 2023-07-20 00:00:00.0 US/Pacific AAA_1
+#Masked#0 2024-07-21 00:00:00.0 US/Pacific AAA_2
+#Masked#1 2024-08-22 00:00:00.0 US/Pacific AAA_1
+#Masked#2 2024-08-23 00:00:00.0 US/Pacific AAA_2
+#Masked#3 2024-08-24 00:00:00.0 US/Pacific AAA_1
+#Masked#4 2024-08-25 00:00:00.0 US/Pacific AAA_2
+#Masked#5 2024-09-24 00:00:00.0 US/Pacific AAA_1
+#Masked#6 2024-09-25 00:00:00.0 US/Pacific AAA_2
+#Masked#7 2024-07-01 00:00:00.0 US/Pacific BBB_1
+#Masked#8 2024-07-02 00:00:00.0 US/Pacific BBB_2
+#Masked#9 2024-07-03 00:00:00.0 US/Pacific BBB_1
+#Masked# 2023-07-21 00:00:00.0 US/Pacific AAA_2
+#Masked#0 2024-07-04 00:00:00.0 US/Pacific BBB_2
+#Masked#1 2024-08-03 00:00:00.0 US/Pacific BBB_1
+#Masked#2 2024-08-04 00:00:00.0 US/Pacific BBB_2
+#Masked#3 2024-08-05 00:00:00.0 US/Pacific BBB_1
+#Masked#4 2024-08-06 00:00:00.0 US/Pacific BBB_2
+#Masked#5 2024-09-05 00:00:00.0 US/Pacific BBB_1
+#Masked#6 2024-09-06 00:00:00.0 US/Pacific BBB_2
+#Masked# 2023-07-23 00:00:00.0 US/Pacific AAA_1
+#Masked# 2023-07-24 00:00:00.0 US/Pacific AAA_2
+#Masked# 2023-08-04 00:00:00.0 US/Pacific BBB_1
+#Masked# 2023-08-05 00:00:00.0 US/Pacific BBB_2
+#Masked# 2023-08-06 00:00:00.0 US/Pacific BBB_1
+#Masked# 2023-08-27 00:00:00.0 US/Pacific BBB_2
+#Masked# 2024-07-20 00:00:00.0 US/Pacific AAA_1
+PREHOOK: query: describe formatted ice_orc
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@ice_orc
+POSTHOOK: query: describe formatted ice_orc
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@ice_orc
+# col_name data_type comment
+event_id int
+event_time timestamp with local time zone
+event_src string
+
+# Partition Transform Information
+# col_name transform_type
+event_src TRUNCATE[3]
+event_time MONTH
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+#### A masked pattern was here ####
+Table Type: EXTERNAL_TABLE
+Table Parameters:
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ EXTERNAL TRUE
+ bucketing_version 2
+ current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"event_id\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"event_time\",\"required\":false,\"type\":\"timestamptz\"},{\"id\":3,\"name\":\"event_src\",\"required\":false,\"type\":\"string\"}]}
+ current-snapshot-id #Masked#
+ current-snapshot-summary
{\"added-data-files\":\"1\",\"added-records\":\"2\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"26\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"13\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\"}
+ current-snapshot-timestamp-ms #Masked#
+ default-partition-spec
{\"spec-id\":1,\"fields\":[{\"name\":\"event_src_trunc\",\"transform\":\"truncate[3]\",\"source-id\":3,\"field-id\":1000},{\"name\":\"event_time_month\",\"transform\":\"month\",\"source-id\":2,\"field-id\":1001}]}
+ format-version 2
+ iceberg.orc.files.only true
+#### A masked pattern was here ####
+ numFiles 13
+ numRows 26
+ parquet.compression zstd
+#### A masked pattern was here ####
+ rawDataSize 0
+ serialization.format 1
+ snapshot-count 13
+ storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
+ table_type ICEBERG
+ totalSize #Masked#
+#### A masked pattern was here ####
+ uuid #Masked#
+ write.delete.mode merge-on-read
+ write.format.default orc
+ write.merge.mode merge-on-read
+ write.update.mode merge-on-read
+
+# Storage Information
+SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe
+InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
+Compressed: No
+Sort Columns: []
+PREHOOK: query: alter table ice_orc COMPACT 'major' and wait
+where (event_src in ('BBB_1', 'BBB_2') and event_time < '2024-09-01 00:00:00')
or
+ (event_src in ('AAA_1', 'AAA_2') and event_time > '2024-08-01 00:00:00')
+PREHOOK: type: ALTERTABLE_COMPACT
+PREHOOK: Input: default@ice_orc
+PREHOOK: Output: default@ice_orc
+POSTHOOK: query: alter table ice_orc COMPACT 'major' and wait
+where (event_src in ('BBB_1', 'BBB_2') and event_time < '2024-09-01 00:00:00')
or
+ (event_src in ('AAA_1', 'AAA_2') and event_time > '2024-08-01 00:00:00')
+POSTHOOK: type: ALTERTABLE_COMPACT
+POSTHOOK: Input: default@ice_orc
+POSTHOOK: Output: default@ice_orc
+PREHOOK: query: select * from ice_orc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_orc
+#### A masked pattern was here ####
+POSTHOOK: query: select * from ice_orc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_orc
+#### A masked pattern was here ####
+#Masked# 2023-07-20 00:00:00.0 US/Pacific AAA_1
+#Masked#0 2024-07-21 00:00:00.0 US/Pacific AAA_2
+#Masked#1 2024-08-22 00:00:00.0 US/Pacific AAA_1
+#Masked#2 2024-08-23 00:00:00.0 US/Pacific AAA_2
+#Masked#3 2024-08-24 00:00:00.0 US/Pacific AAA_1
+#Masked#4 2024-08-25 00:00:00.0 US/Pacific AAA_2
+#Masked#5 2024-09-24 00:00:00.0 US/Pacific AAA_1
+#Masked#6 2024-09-25 00:00:00.0 US/Pacific AAA_2
+#Masked#7 2024-07-01 00:00:00.0 US/Pacific BBB_1
+#Masked#8 2024-07-02 00:00:00.0 US/Pacific BBB_2
+#Masked#9 2024-07-03 00:00:00.0 US/Pacific BBB_1
+#Masked# 2023-07-21 00:00:00.0 US/Pacific AAA_2
+#Masked#0 2024-07-04 00:00:00.0 US/Pacific BBB_2
+#Masked#1 2024-08-03 00:00:00.0 US/Pacific BBB_1
+#Masked#2 2024-08-04 00:00:00.0 US/Pacific BBB_2
+#Masked#3 2024-08-05 00:00:00.0 US/Pacific BBB_1
+#Masked#4 2024-08-06 00:00:00.0 US/Pacific BBB_2
+#Masked#5 2024-09-05 00:00:00.0 US/Pacific BBB_1
+#Masked#6 2024-09-06 00:00:00.0 US/Pacific BBB_2
+#Masked# 2023-07-23 00:00:00.0 US/Pacific AAA_1
+#Masked# 2023-07-24 00:00:00.0 US/Pacific AAA_2
+#Masked# 2023-08-04 00:00:00.0 US/Pacific BBB_1
+#Masked# 2023-08-05 00:00:00.0 US/Pacific BBB_2
+#Masked# 2023-08-06 00:00:00.0 US/Pacific BBB_1
+#Masked# 2023-08-27 00:00:00.0 US/Pacific BBB_2
+#Masked# 2024-07-20 00:00:00.0 US/Pacific AAA_1
+PREHOOK: query: describe formatted ice_orc
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@ice_orc
+POSTHOOK: query: describe formatted ice_orc
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@ice_orc
+# col_name data_type comment
+event_id int
+event_time timestamp with local time zone
+event_src string
+
+# Partition Transform Information
+# col_name transform_type
+event_src TRUNCATE[3]
+event_time MONTH
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+#### A masked pattern was here ####
+Table Type: EXTERNAL_TABLE
+Table Parameters:
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ EXTERNAL TRUE
+ bucketing_version 2
+ current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"event_id\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"event_time\",\"required\":false,\"type\":\"timestamptz\"},{\"id\":3,\"name\":\"event_src\",\"required\":false,\"type\":\"string\"}]}
+ current-snapshot-id #Masked#
+ current-snapshot-summary
{\"added-data-files\":\"2\",\"deleted-data-files\":\"4\",\"added-records\":\"8\",\"deleted-records\":\"8\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"4\",\"total-records\":\"26\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"8\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\"}
+ current-snapshot-timestamp-ms #Masked#
+ default-partition-spec
{\"spec-id\":1,\"fields\":[{\"name\":\"event_src_trunc\",\"transform\":\"truncate[3]\",\"source-id\":3,\"field-id\":1000},{\"name\":\"event_time_month\",\"transform\":\"month\",\"source-id\":2,\"field-id\":1001}]}
+ format-version 2
+ iceberg.orc.files.only true
+#### A masked pattern was here ####
+ numFiles 8
+ numRows 26
+ parquet.compression zstd
+#### A masked pattern was here ####
+ rawDataSize 0
+ serialization.format 1
+ snapshot-count 18
+ storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
+ table_type ICEBERG
+ totalSize #Masked#
+#### A masked pattern was here ####
+ uuid #Masked#
+ write.delete.mode merge-on-read
+ write.format.default orc
+ write.merge.mode merge-on-read
+ write.update.mode merge-on-read
+
+# Storage Information
+SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe
+InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
+Compressed: No
+Sort Columns: []
+PREHOOK: query: show compactions order by 'partition'
+PREHOOK: type: SHOW COMPACTIONS
+POSTHOOK: query: show compactions order by 'partition'
+POSTHOOK: type: SHOW COMPACTIONS
+CompactionId Database Table Partition Type State Worker
host Worker Enqueue Time Start Time Duration(ms) HadoopJobId
Error message Initiator host Initiator Pool name TxnId Next
TxnId Commit Time Highest WriteId
+#Masked# default ice_orc event_src_trunc=AAA/event_time_month=2024-08
MAJOR succeeded #Masked# manual default 0 0 0
---
+#Masked# default ice_orc event_src_trunc=AAA/event_time_month=2024-09
MAJOR succeeded #Masked# manual default 0 0 0
---
+#Masked# default ice_orc event_src_trunc=BBB/event_time_month=2024-07
MAJOR succeeded #Masked# manual default 0 0 0
---
+#Masked# default ice_orc event_src_trunc=BBB/event_time_month=2024-08
MAJOR succeeded #Masked# manual default 0 0 0
---
+#Masked# default ice_orc --- MAJOR succeeded #Masked#
manual default 0 0 0 ---
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q.out
similarity index 63%
copy from
iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out
copy to
iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q.out
index d9dee1d9902..d70e2b958a9 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q.out
@@ -22,179 +22,133 @@ tblproperties ('format-version'='2')
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc VALUES ('fn1','ln1', 1, 10, 100)
+PREHOOK: query: insert into ice_orc VALUES
+ ('fn1','ln1', 1, 10, 100),
+ ('fn2','ln2', 1, 10, 100)
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc VALUES ('fn1','ln1', 1, 10, 100)
+POSTHOOK: query: insert into ice_orc VALUES
+ ('fn1','ln1', 1, 10, 100),
+ ('fn2','ln2', 1, 10, 100)
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc VALUES ('fn2','ln2', 1, 10, 100)
+PREHOOK: query: insert into ice_orc VALUES
+ ('fn3','ln3', 2, 11, 100),
+ ('fn4','ln4', 2, 11, 100)
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc VALUES ('fn2','ln2', 1, 10, 100)
+POSTHOOK: query: insert into ice_orc VALUES
+ ('fn3','ln3', 2, 11, 100),
+ ('fn4','ln4', 2, 11, 100)
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc VALUES ('fn3','ln3', 1, 11, 100)
+PREHOOK: query: insert into ice_orc VALUES
+ ('fn5','ln5', 3, 12, 100),
+ ('fn6','ln6', 3, 12, 100)
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc VALUES ('fn3','ln3', 1, 11, 100)
+POSTHOOK: query: insert into ice_orc VALUES
+ ('fn5','ln5', 3, 12, 100),
+ ('fn6','ln6', 3, 12, 100)
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: alter table ice_orc set partition spec(company_id, dept_id)
-PREHOOK: type: ALTERTABLE_SETPARTSPEC
-PREHOOK: Input: default@ice_orc
-POSTHOOK: query: alter table ice_orc set partition spec(company_id, dept_id)
-POSTHOOK: type: ALTERTABLE_SETPARTSPEC
-POSTHOOK: Input: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc VALUES ('fn4','ln4', 1, 11, 100)
-PREHOOK: type: QUERY
-PREHOOK: Input: _dummy_database@_dummy_table
-PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc VALUES ('fn4','ln4', 1, 11, 100)
-POSTHOOK: type: QUERY
-POSTHOOK: Input: _dummy_database@_dummy_table
-POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc VALUES ('fn5','ln5', 2, 20, 100)
-PREHOOK: type: QUERY
-PREHOOK: Input: _dummy_database@_dummy_table
-PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc VALUES ('fn5','ln5', 2, 20, 100)
-POSTHOOK: type: QUERY
-POSTHOOK: Input: _dummy_database@_dummy_table
-POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc VALUES ('fn6','ln6', 2, 20, 100)
+PREHOOK: query: insert into ice_orc VALUES
+ ('fn7','ln7', 4, 13, 100),
+ ('fn8','ln8', 4, 13, 100)
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc VALUES ('fn6','ln6', 2, 20, 100)
+POSTHOOK: query: insert into ice_orc VALUES
+ ('fn7','ln7', 4, 13, 100),
+ ('fn8','ln8', 4, 13, 100)
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: alter table ice_orc set partition spec(company_id, dept_id,
team_id)
+PREHOOK: query: alter table ice_orc set partition spec(company_id, dept_id)
PREHOOK: type: ALTERTABLE_SETPARTSPEC
PREHOOK: Input: default@ice_orc
-POSTHOOK: query: alter table ice_orc set partition spec(company_id, dept_id,
team_id)
+POSTHOOK: query: alter table ice_orc set partition spec(company_id, dept_id)
POSTHOOK: type: ALTERTABLE_SETPARTSPEC
POSTHOOK: Input: default@ice_orc
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc VALUES ('fn7','ln7', 2, 21, 100)
+PREHOOK: query: insert into ice_orc VALUES
+ ('fn9', 'ln9', 1, 10, 100),
+ ('fn10','ln10', 1, 10, 100)
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc VALUES ('fn7','ln7', 2, 21, 100)
+POSTHOOK: query: insert into ice_orc VALUES
+ ('fn9', 'ln9', 1, 10, 100),
+ ('fn10','ln10', 1, 10, 100)
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc VALUES ('fn8','ln8', 2, 21, 100)
+PREHOOK: query: insert into ice_orc VALUES
+ ('fn11','ln11', 2, 11, 100),
+ ('fn12','ln12', 2, 11, 100)
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc VALUES ('fn8','ln8', 2, 21, 100)
+POSTHOOK: query: insert into ice_orc VALUES
+ ('fn11','ln11', 2, 11, 100),
+ ('fn12','ln12', 2, 11, 100)
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: update ice_orc set last_name = 'ln1a' where first_name='fn1'
-PREHOOK: type: QUERY
-PREHOOK: Input: default@ice_orc
-PREHOOK: Output: default@ice_orc
-PREHOOK: Output: default@ice_orc
-POSTHOOK: query: update ice_orc set last_name = 'ln1a' where first_name='fn1'
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-PREHOOK: query: update ice_orc set last_name = 'ln2a' where first_name='fn2'
-PREHOOK: type: QUERY
-PREHOOK: Input: default@ice_orc
-PREHOOK: Output: default@ice_orc
-PREHOOK: Output: default@ice_orc
-POSTHOOK: query: update ice_orc set last_name = 'ln2a' where first_name='fn2'
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-PREHOOK: query: update ice_orc set last_name = 'ln3a' where first_name='fn3'
-PREHOOK: type: QUERY
-PREHOOK: Input: default@ice_orc
-PREHOOK: Output: default@ice_orc
-PREHOOK: Output: default@ice_orc
-POSTHOOK: query: update ice_orc set last_name = 'ln3a' where first_name='fn3'
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-PREHOOK: query: update ice_orc set last_name = 'ln4a' where first_name='fn4'
+PREHOOK: query: insert into ice_orc VALUES
+ ('fn13','ln13', 3, 12, 100),
+ ('fn14','ln14', 3, 12, 100)
PREHOOK: type: QUERY
-PREHOOK: Input: default@ice_orc
-PREHOOK: Output: default@ice_orc
+PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: update ice_orc set last_name = 'ln4a' where first_name='fn4'
+POSTHOOK: query: insert into ice_orc VALUES
+ ('fn13','ln13', 3, 12, 100),
+ ('fn14','ln14', 3, 12, 100)
POSTHOOK: type: QUERY
-POSTHOOK: Input: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-PREHOOK: query: alter table ice_orc set partition spec(company_id, dept_id)
-PREHOOK: type: ALTERTABLE_SETPARTSPEC
-PREHOOK: Input: default@ice_orc
-POSTHOOK: query: alter table ice_orc set partition spec(company_id, dept_id)
-POSTHOOK: type: ALTERTABLE_SETPARTSPEC
-POSTHOOK: Input: default@ice_orc
+POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: update ice_orc set last_name = 'ln5a' where first_name='fn5'
+PREHOOK: query: insert into ice_orc VALUES
+ ('fn15','ln15', 4, 13, 100),
+ ('fn16','ln16', 4, 13, 100)
PREHOOK: type: QUERY
-PREHOOK: Input: default@ice_orc
-PREHOOK: Output: default@ice_orc
+PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: update ice_orc set last_name = 'ln5a' where first_name='fn5'
+POSTHOOK: query: insert into ice_orc VALUES
+ ('fn15','ln15', 4, 13, 100),
+ ('fn16','ln16', 4, 13, 100)
POSTHOOK: type: QUERY
-POSTHOOK: Input: default@ice_orc
-POSTHOOK: Output: default@ice_orc
+POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: update ice_orc set last_name = 'ln6a' where first_name='fn6'
+PREHOOK: query: delete from ice_orc where last_name in ('ln1', 'ln9')
PREHOOK: type: QUERY
PREHOOK: Input: default@ice_orc
PREHOOK: Output: default@ice_orc
-PREHOOK: Output: default@ice_orc
-POSTHOOK: query: update ice_orc set last_name = 'ln6a' where first_name='fn6'
+POSTHOOK: query: delete from ice_orc where last_name in ('ln1', 'ln9')
POSTHOOK: type: QUERY
POSTHOOK: Input: default@ice_orc
POSTHOOK: Output: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-PREHOOK: query: update ice_orc set last_name = 'ln7a' where first_name='fn7'
+PREHOOK: query: delete from ice_orc where last_name in ('ln3', 'ln11')
PREHOOK: type: QUERY
PREHOOK: Input: default@ice_orc
PREHOOK: Output: default@ice_orc
-PREHOOK: Output: default@ice_orc
-POSTHOOK: query: update ice_orc set last_name = 'ln7a' where first_name='fn7'
+POSTHOOK: query: delete from ice_orc where last_name in ('ln3', 'ln11')
POSTHOOK: type: QUERY
POSTHOOK: Input: default@ice_orc
POSTHOOK: Output: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-PREHOOK: query: update ice_orc set last_name = 'ln8a' where first_name='fn8'
+PREHOOK: query: delete from ice_orc where last_name in ('ln5', 'ln13')
PREHOOK: type: QUERY
PREHOOK: Input: default@ice_orc
PREHOOK: Output: default@ice_orc
-PREHOOK: Output: default@ice_orc
-POSTHOOK: query: update ice_orc set last_name = 'ln8a' where first_name='fn8'
+POSTHOOK: query: delete from ice_orc where last_name in ('ln5', 'ln13')
POSTHOOK: type: QUERY
POSTHOOK: Input: default@ice_orc
POSTHOOK: Output: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-PREHOOK: query: delete from ice_orc where last_name in ('ln1a', 'ln8a')
-PREHOOK: type: QUERY
-PREHOOK: Input: default@ice_orc
-#### A masked pattern was here ####
-POSTHOOK: query: delete from ice_orc where last_name in ('ln1a', 'ln8a')
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@ice_orc
-#### A masked pattern was here ####
PREHOOK: query: select * from ice_orc
PREHOOK: type: QUERY
PREHOOK: Input: default@ice_orc
@@ -203,12 +157,16 @@ POSTHOOK: query: select * from ice_orc
POSTHOOK: type: QUERY
POSTHOOK: Input: default@ice_orc
#### A masked pattern was here ####
-fn2 ln2a 1 10 100
-fn3 ln3a 1 11 100
-fn4 ln4a 1 11 100
-fn5 ln5a 2 20 100
-fn6 ln6a 2 20 100
-fn7 ln7a 2 21 100
+fn10 ln10 1 10 100
+fn12 ln12 2 11 100
+fn14 ln14 3 12 100
+fn15 ln15 4 13 100
+fn16 ln16 4 13 100
+fn2 ln2 1 10 100
+fn4 ln4 2 11 100
+fn6 ln6 3 12 100
+fn7 ln7 4 13 100
+fn8 ln8 4 13 100
PREHOOK: query: describe formatted ice_orc
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_orc
@@ -239,19 +197,19 @@ Table Parameters:
bucketing_version 2
current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]}
current-snapshot-id #Masked#
- current-snapshot-summary
{\"deleted-data-files\":\"2\",\"deleted-records\":\"2\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"2\",\"total-records\":\"14\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"14\",\"total-delete-files\":\"8\",\"total-position-deletes\":\"8\",\"total-equality-deletes\":\"0\"}
+ current-snapshot-summary
{\"added-position-delete-files\":\"2\",\"added-delete-files\":\"2\",\"added-files-size\":\"#Masked#\",\"added-position-deletes\":\"2\",\"changed-partition-count\":\"2\",\"total-records\":\"16\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"8\",\"total-delete-files\":\"6\",\"total-position-deletes\":\"6\",\"total-equality-deletes\":\"0\"}
current-snapshot-timestamp-ms #Masked#
default-partition-spec
{\"spec-id\":1,\"fields\":[{\"name\":\"company_id\",\"transform\":\"identity\",\"source-id\":5,\"field-id\":1000},{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1001}]}
format-version 2
iceberg.orc.files.only true
#### A masked pattern was here ####
- numFiles 14
- numRows 14
+ numFiles 8
+ numRows 10
parquet.compression zstd
#### A masked pattern was here ####
rawDataSize 0
serialization.format 1
- snapshot-count 17
+ snapshot-count 11
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
totalSize #Masked#
@@ -268,11 +226,11 @@ InputFormat:
org.apache.iceberg.mr.hive.HiveIcebergInputFormat
OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
Compressed: No
Sort Columns: []
-PREHOOK: query: explain alter table ice_orc COMPACT 'major' and wait
+PREHOOK: query: explain alter table ice_orc COMPACT 'major' and wait where
team_id=10 or first_name in ('fn3', 'fn11') or last_name in ('ln7', 'ln15')
PREHOOK: type: ALTERTABLE_COMPACT
PREHOOK: Input: default@ice_orc
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: explain alter table ice_orc COMPACT 'major' and wait
+POSTHOOK: query: explain alter table ice_orc COMPACT 'major' and wait where
team_id=10 or first_name in ('fn3', 'fn11') or last_name in ('ln7', 'ln15')
POSTHOOK: type: ALTERTABLE_COMPACT
POSTHOOK: Input: default@ice_orc
POSTHOOK: Output: default@ice_orc
@@ -288,11 +246,11 @@ STAGE PLANS:
table name: default.ice_orc
blocking: true
-PREHOOK: query: alter table ice_orc COMPACT 'major' and wait
+PREHOOK: query: alter table ice_orc COMPACT 'major' and wait where team_id=10
or first_name in ('fn3', 'fn11') or last_name in ('ln7', 'ln15')
PREHOOK: type: ALTERTABLE_COMPACT
PREHOOK: Input: default@ice_orc
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: alter table ice_orc COMPACT 'major' and wait
+POSTHOOK: query: alter table ice_orc COMPACT 'major' and wait where team_id=10
or first_name in ('fn3', 'fn11') or last_name in ('ln7', 'ln15')
POSTHOOK: type: ALTERTABLE_COMPACT
POSTHOOK: Input: default@ice_orc
POSTHOOK: Output: default@ice_orc
@@ -304,12 +262,16 @@ POSTHOOK: query: select * from ice_orc
POSTHOOK: type: QUERY
POSTHOOK: Input: default@ice_orc
#### A masked pattern was here ####
-fn2 ln2a 1 10 100
-fn3 ln3a 1 11 100
-fn4 ln4a 1 11 100
-fn5 ln5a 2 20 100
-fn6 ln6a 2 20 100
-fn7 ln7a 2 21 100
+fn10 ln10 1 10 100
+fn12 ln12 2 11 100
+fn14 ln14 3 12 100
+fn15 ln15 4 13 100
+fn16 ln16 4 13 100
+fn2 ln2 1 10 100
+fn4 ln4 2 11 100
+fn6 ln6 3 12 100
+fn7 ln7 4 13 100
+fn8 ln8 4 13 100
PREHOOK: query: describe formatted ice_orc
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_orc
@@ -340,19 +302,19 @@ Table Parameters:
bucketing_version 2
current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]}
current-snapshot-id #Masked#
- current-snapshot-summary
{\"added-data-files\":\"1\",\"deleted-data-files\":\"8\",\"removed-position-delete-files\":\"5\",\"removed-delete-files\":\"5\",\"added-records\":\"3\",\"deleted-records\":\"8\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"removed-position-deletes\":\"5\",\"changed-partition-count\":\"5\",\"total-records\":\"6\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"2\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"
[...]
+ current-snapshot-summary
{\"added-data-files\":\"4\",\"deleted-data-files\":\"4\",\"removed-position-delete-files\":\"3\",\"removed-delete-files\":\"3\",\"added-records\":\"5\",\"deleted-records\":\"8\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"removed-position-deletes\":\"3\",\"changed-partition-count\":\"5\",\"total-records\":\"11\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"8\",\"total-delete-files\":\"1\",\"total-position-deletes\":\"1\",\
[...]
current-snapshot-timestamp-ms #Masked#
default-partition-spec
{\"spec-id\":1,\"fields\":[{\"name\":\"company_id\",\"transform\":\"identity\",\"source-id\":5,\"field-id\":1000},{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1001}]}
format-version 2
iceberg.orc.files.only true
#### A masked pattern was here ####
- numFiles 2
- numRows 6
+ numFiles 8
+ numRows 10
parquet.compression zstd
#### A masked pattern was here ####
rawDataSize 0
serialization.format 1
- snapshot-count 20
+ snapshot-count 15
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
totalSize #Masked#
@@ -369,11 +331,12 @@ InputFormat:
org.apache.iceberg.mr.hive.HiveIcebergInputFormat
OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
Compressed: No
Sort Columns: []
-PREHOOK: query: show compactions
+PREHOOK: query: show compactions order by 'partition'
PREHOOK: type: SHOW COMPACTIONS
-POSTHOOK: query: show compactions
+POSTHOOK: query: show compactions order by 'partition'
POSTHOOK: type: SHOW COMPACTIONS
CompactionId Database Table Partition Type State Worker
host Worker Enqueue Time Start Time Duration(ms) HadoopJobId
Error message Initiator host Initiator Pool name TxnId Next
TxnId Commit Time Highest WriteId
#Masked# default ice_orc company_id=100/dept_id=1 MAJOR
succeeded #Masked# manual default 0 0 0 ---
#Masked# default ice_orc company_id=100/dept_id=2 MAJOR
succeeded #Masked# manual default 0 0 0 ---
+#Masked# default ice_orc company_id=100/dept_id=4 MAJOR
succeeded #Masked# manual default 0 0 0 ---
#Masked# default ice_orc --- MAJOR succeeded #Masked#
manual default 0 0 0 ---
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partitioned.q.out
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partitioned.q.out
index 781bb41dd14..cf2d3936b57 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partitioned.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partitioned.q.out
@@ -316,9 +316,9 @@ InputFormat:
org.apache.iceberg.mr.hive.HiveIcebergInputFormat
OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
Compressed: No
Sort Columns: []
-PREHOOK: query: show compactions
+PREHOOK: query: show compactions order by 'partition'
PREHOOK: type: SHOW COMPACTIONS
-POSTHOOK: query: show compactions
+POSTHOOK: query: show compactions order by 'partition'
POSTHOOK: type: SHOW COMPACTIONS
CompactionId Database Table Partition Type State Worker
host Worker Enqueue Time Start Time Duration(ms) HadoopJobId
Error message Initiator host Initiator Pool name TxnId Next
TxnId Commit Time Highest WriteId
#Masked# default ice_orc dept_id=1 MAJOR succeeded
#Masked# manual default 0 0 0 ---
@@ -647,12 +647,12 @@ InputFormat:
org.apache.iceberg.mr.hive.HiveIcebergInputFormat
OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
Compressed: No
Sort Columns: []
-PREHOOK: query: show compactions
+PREHOOK: query: show compactions order by 'partition'
PREHOOK: type: SHOW COMPACTIONS
-POSTHOOK: query: show compactions
+POSTHOOK: query: show compactions order by 'partition'
POSTHOOK: type: SHOW COMPACTIONS
CompactionId Database Table Partition Type State Worker
host Worker Enqueue Time Start Time Duration(ms) HadoopJobId
Error message Initiator host Initiator Pool name TxnId Next
TxnId Commit Time Highest WriteId
#Masked# default ice_orc dept_id=1 MAJOR succeeded
#Masked# manual default 0 0 0 ---
-#Masked# default ice_orc dept_id=2 MAJOR succeeded
#Masked# manual default 0 0 0 ---
#Masked# default ice_orc dept_id=1 MAJOR succeeded
#Masked# manual default 0 0 0 ---
#Masked# default ice_orc dept_id=2 MAJOR succeeded
#Masked# manual default 0 0 0 ---
+#Masked# default ice_orc dept_id=2 MAJOR succeeded
#Masked# manual default 0 0 0 ---
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_schema_evolution.q.out
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_schema_evolution.q.out
index 03deb181cf6..ecc8fe3c36f 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_schema_evolution.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_schema_evolution.q.out
@@ -354,9 +354,9 @@ InputFormat:
org.apache.iceberg.mr.hive.HiveIcebergInputFormat
OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
Compressed: No
Sort Columns: []
-PREHOOK: query: show compactions
+PREHOOK: query: show compactions order by 'partition'
PREHOOK: type: SHOW COMPACTIONS
-POSTHOOK: query: show compactions
+POSTHOOK: query: show compactions order by 'partition'
POSTHOOK: type: SHOW COMPACTIONS
CompactionId Database Table Partition Type State Worker
host Worker Enqueue Time Start Time Duration(ms) HadoopJobId
Error message Initiator host Initiator Pool name TxnId Next
TxnId Commit Time Highest WriteId
#Masked# default ice_orc dept_id=1 MAJOR succeeded
#Masked# manual default 0 0 0 ---
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition.q.out
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition.q.out
index 41dd8dc89fa..4933952db28 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition.q.out
@@ -332,9 +332,9 @@ InputFormat:
org.apache.iceberg.mr.hive.HiveIcebergInputFormat
OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
Compressed: No
Sort Columns: []
-PREHOOK: query: show compactions
+PREHOOK: query: show compactions order by 'partition'
PREHOOK: type: SHOW COMPACTIONS
-POSTHOOK: query: show compactions
+POSTHOOK: query: show compactions order by 'partition'
POSTHOOK: type: SHOW COMPACTIONS
CompactionId Database Table Partition Type State Worker
host Worker Enqueue Time Start Time Duration(ms) HadoopJobId
Error message Initiator host Initiator Pool name TxnId Next
TxnId Commit Time Highest WriteId
#Masked# default ice_orc_wo_evo
dept_id=1/city=London/registration_date=2024-03-11 MAJOR succeeded
#Masked# manual default 0 0 0 ---
@@ -442,9 +442,9 @@ InputFormat:
org.apache.iceberg.mr.hive.HiveIcebergInputFormat
OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
Compressed: No
Sort Columns: []
-PREHOOK: query: show compactions
+PREHOOK: query: show compactions order by 'partition'
PREHOOK: type: SHOW COMPACTIONS
-POSTHOOK: query: show compactions
+POSTHOOK: query: show compactions order by 'partition'
POSTHOOK: type: SHOW COMPACTIONS
CompactionId Database Table Partition Type State Worker
host Worker Enqueue Time Start Time Duration(ms) HadoopJobId
Error message Initiator host Initiator Pool name TxnId Next
TxnId Commit Time Highest WriteId
#Masked# default ice_orc_wo_evo
dept_id=1/city=London/registration_date=2024-03-11 MAJOR succeeded
#Masked# manual default 0 0 0 ---
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution.q.out
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution.q.out
index 806675b676f..e70234be394 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution.q.out
@@ -639,12 +639,12 @@ InputFormat:
org.apache.iceberg.mr.hive.HiveIcebergInputFormat
OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
Compressed: No
Sort Columns: []
-PREHOOK: query: show compactions
+PREHOOK: query: show compactions order by 'partition'
PREHOOK: type: SHOW COMPACTIONS
-POSTHOOK: query: show compactions
+POSTHOOK: query: show compactions order by 'partition'
POSTHOOK: type: SHOW COMPACTIONS
CompactionId Database Table Partition Type State Worker
host Worker Enqueue Time Start Time Duration(ms) HadoopJobId
Error message Initiator host Initiator Pool name TxnId Next
TxnId Commit Time Highest WriteId
-#Masked# default ice_orc
dept_id=1/city=London/registration_date=2024-03-11 MAJOR succeeded
#Masked# manual default 0 0 0 ---
-#Masked# default ice_orc
dept_id=2/city=Paris/registration_date=2024-02-16 MAJOR succeeded
#Masked# manual default 0 0 0 ---
#Masked# default ice_orc dept_id=1/city=London MAJOR succeeded
#Masked# manual default 0 0 0 ---
+#Masked# default ice_orc
dept_id=1/city=London/registration_date=2024-03-11 MAJOR succeeded
#Masked# manual default 0 0 0 ---
#Masked# default ice_orc dept_id=2/city=Paris MAJOR succeeded
#Masked# manual default 0 0 0 ---
+#Masked# default ice_orc
dept_id=2/city=Paris/registration_date=2024-02-16 MAJOR succeeded
#Masked# manual default 0 0 0 ---
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution2.q.out
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution2.q.out
index d8bc84569fb..c10239b40ae 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution2.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution2.q.out
@@ -443,3 +443,10 @@ InputFormat:
org.apache.iceberg.mr.hive.HiveIcebergInputFormat
OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
Compressed: No
Sort Columns: []
+PREHOOK: query: show compactions order by 'partition'
+PREHOOK: type: SHOW COMPACTIONS
+POSTHOOK: query: show compactions order by 'partition'
+POSTHOOK: type: SHOW COMPACTIONS
+CompactionId Database Table Partition Type State Worker
host Worker Enqueue Time Start Time Duration(ms) HadoopJobId
Error message Initiator host Initiator Pool name TxnId Next
TxnId Commit Time Highest WriteId
+#Masked# default ice_orc a=a MAJOR succeeded #Masked#
manual default 0 0 0 ---
+#Masked# default ice_orc b=1 MAJOR succeeded #Masked#
manual default 0 0 0 ---
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_unpartitioned_w_filter.q.out
similarity index 62%
copy from
iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out
copy to
iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_unpartitioned_w_filter.q.out
index d9dee1d9902..7060f8917fe 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_unpartitioned_w_filter.q.out
@@ -1,10 +1,7 @@
PREHOOK: query: create table ice_orc (
first_name string,
- last_name string,
- dept_id bigint,
- team_id bigint
+ last_name string
)
-partitioned by (company_id bigint)
stored by iceberg stored as orc
tblproperties ('format-version'='2')
PREHOOK: type: CREATETABLE
@@ -12,91 +9,66 @@ PREHOOK: Output: database:default
PREHOOK: Output: default@ice_orc
POSTHOOK: query: create table ice_orc (
first_name string,
- last_name string,
- dept_id bigint,
- team_id bigint
+ last_name string
)
-partitioned by (company_id bigint)
stored by iceberg stored as orc
tblproperties ('format-version'='2')
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc VALUES ('fn1','ln1', 1, 10, 100)
+PREHOOK: query: insert into ice_orc VALUES ('fn1','ln1')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc VALUES ('fn1','ln1', 1, 10, 100)
+POSTHOOK: query: insert into ice_orc VALUES ('fn1','ln1')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc VALUES ('fn2','ln2', 1, 10, 100)
+PREHOOK: query: insert into ice_orc VALUES ('fn2','ln2')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc VALUES ('fn2','ln2', 1, 10, 100)
+POSTHOOK: query: insert into ice_orc VALUES ('fn2','ln2')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc VALUES ('fn3','ln3', 1, 11, 100)
+PREHOOK: query: insert into ice_orc VALUES ('fn3','ln3')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc VALUES ('fn3','ln3', 1, 11, 100)
+POSTHOOK: query: insert into ice_orc VALUES ('fn3','ln3')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: alter table ice_orc set partition spec(company_id, dept_id)
-PREHOOK: type: ALTERTABLE_SETPARTSPEC
-PREHOOK: Input: default@ice_orc
-POSTHOOK: query: alter table ice_orc set partition spec(company_id, dept_id)
-POSTHOOK: type: ALTERTABLE_SETPARTSPEC
-POSTHOOK: Input: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc VALUES ('fn4','ln4', 1, 11, 100)
+PREHOOK: query: insert into ice_orc VALUES ('fn4','ln4')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc VALUES ('fn4','ln4', 1, 11, 100)
+POSTHOOK: query: insert into ice_orc VALUES ('fn4','ln4')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc VALUES ('fn5','ln5', 2, 20, 100)
+PREHOOK: query: insert into ice_orc VALUES ('fn5','ln5')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc VALUES ('fn5','ln5', 2, 20, 100)
+POSTHOOK: query: insert into ice_orc VALUES ('fn5','ln5')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc VALUES ('fn6','ln6', 2, 20, 100)
+PREHOOK: query: insert into ice_orc VALUES ('fn6','ln6')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc VALUES ('fn6','ln6', 2, 20, 100)
+POSTHOOK: query: insert into ice_orc VALUES ('fn6','ln6')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: alter table ice_orc set partition spec(company_id, dept_id,
team_id)
-PREHOOK: type: ALTERTABLE_SETPARTSPEC
-PREHOOK: Input: default@ice_orc
-POSTHOOK: query: alter table ice_orc set partition spec(company_id, dept_id,
team_id)
-POSTHOOK: type: ALTERTABLE_SETPARTSPEC
-POSTHOOK: Input: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc VALUES ('fn7','ln7', 2, 21, 100)
+PREHOOK: query: insert into ice_orc VALUES ('fn7','ln7')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc VALUES ('fn7','ln7', 2, 21, 100)
-POSTHOOK: type: QUERY
-POSTHOOK: Input: _dummy_database@_dummy_table
-POSTHOOK: Output: default@ice_orc
-PREHOOK: query: insert into ice_orc VALUES ('fn8','ln8', 2, 21, 100)
-PREHOOK: type: QUERY
-PREHOOK: Input: _dummy_database@_dummy_table
-PREHOOK: Output: default@ice_orc
-POSTHOOK: query: insert into ice_orc VALUES ('fn8','ln8', 2, 21, 100)
+POSTHOOK: query: insert into ice_orc VALUES ('fn7','ln7')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_orc
@@ -140,13 +112,6 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@ice_orc
POSTHOOK: Output: default@ice_orc
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: alter table ice_orc set partition spec(company_id, dept_id)
-PREHOOK: type: ALTERTABLE_SETPARTSPEC
-PREHOOK: Input: default@ice_orc
-POSTHOOK: query: alter table ice_orc set partition spec(company_id, dept_id)
-POSTHOOK: type: ALTERTABLE_SETPARTSPEC
-POSTHOOK: Input: default@ice_orc
-POSTHOOK: Output: default@ice_orc
PREHOOK: query: update ice_orc set last_name = 'ln5a' where first_name='fn5'
PREHOOK: type: QUERY
PREHOOK: Input: default@ice_orc
@@ -177,21 +142,11 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@ice_orc
POSTHOOK: Output: default@ice_orc
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: update ice_orc set last_name = 'ln8a' where first_name='fn8'
-PREHOOK: type: QUERY
-PREHOOK: Input: default@ice_orc
-PREHOOK: Output: default@ice_orc
-PREHOOK: Output: default@ice_orc
-POSTHOOK: query: update ice_orc set last_name = 'ln8a' where first_name='fn8'
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-POSTHOOK: Output: default@ice_orc
-PREHOOK: query: delete from ice_orc where last_name in ('ln1a', 'ln8a')
+PREHOOK: query: delete from ice_orc where last_name in ('ln5a', 'ln6a', 'ln7a')
PREHOOK: type: QUERY
PREHOOK: Input: default@ice_orc
#### A masked pattern was here ####
-POSTHOOK: query: delete from ice_orc where last_name in ('ln1a', 'ln8a')
+POSTHOOK: query: delete from ice_orc where last_name in ('ln5a', 'ln6a',
'ln7a')
POSTHOOK: type: QUERY
POSTHOOK: Input: default@ice_orc
#### A masked pattern was here ####
@@ -203,12 +158,10 @@ POSTHOOK: query: select * from ice_orc
POSTHOOK: type: QUERY
POSTHOOK: Input: default@ice_orc
#### A masked pattern was here ####
-fn2 ln2a 1 10 100
-fn3 ln3a 1 11 100
-fn4 ln4a 1 11 100
-fn5 ln5a 2 20 100
-fn6 ln6a 2 20 100
-fn7 ln7a 2 21 100
+fn1 ln1a
+fn2 ln2a
+fn3 ln3a
+fn4 ln4a
PREHOOK: query: describe formatted ice_orc
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_orc
@@ -218,14 +171,6 @@ POSTHOOK: Input: default@ice_orc
# col_name data_type comment
first_name string
last_name string
-dept_id bigint
-team_id bigint
-company_id bigint
-
-# Partition Transform Information
-# col_name transform_type
-company_id IDENTITY
-dept_id IDENTITY
# Detailed Table Information
Database: default
@@ -237,21 +182,20 @@ Table Parameters:
COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
EXTERNAL TRUE
bucketing_version 2
- current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]}
+ current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"}]}
current-snapshot-id #Masked#
- current-snapshot-summary
{\"deleted-data-files\":\"2\",\"deleted-records\":\"2\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"2\",\"total-records\":\"14\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"14\",\"total-delete-files\":\"8\",\"total-position-deletes\":\"8\",\"total-equality-deletes\":\"0\"}
+ current-snapshot-summary
{\"deleted-data-files\":\"3\",\"deleted-records\":\"3\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"11\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"11\",\"total-delete-files\":\"7\",\"total-position-deletes\":\"7\",\"total-equality-deletes\":\"0\"}
current-snapshot-timestamp-ms #Masked#
- default-partition-spec
{\"spec-id\":1,\"fields\":[{\"name\":\"company_id\",\"transform\":\"identity\",\"source-id\":5,\"field-id\":1000},{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1001}]}
format-version 2
iceberg.orc.files.only true
#### A masked pattern was here ####
- numFiles 14
- numRows 14
+ numFiles 11
+ numRows 11
parquet.compression zstd
#### A masked pattern was here ####
rawDataSize 0
serialization.format 1
- snapshot-count 17
+ snapshot-count 15
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
totalSize #Masked#
@@ -268,11 +212,11 @@ InputFormat:
org.apache.iceberg.mr.hive.HiveIcebergInputFormat
OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
Compressed: No
Sort Columns: []
-PREHOOK: query: explain alter table ice_orc COMPACT 'major' and wait
+PREHOOK: query: explain alter table ice_orc COMPACT 'major' and wait where
last_name in ('ln1a', 'ln2a')
PREHOOK: type: ALTERTABLE_COMPACT
PREHOOK: Input: default@ice_orc
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: explain alter table ice_orc COMPACT 'major' and wait
+POSTHOOK: query: explain alter table ice_orc COMPACT 'major' and wait where
last_name in ('ln1a', 'ln2a')
POSTHOOK: type: ALTERTABLE_COMPACT
POSTHOOK: Input: default@ice_orc
POSTHOOK: Output: default@ice_orc
@@ -288,11 +232,11 @@ STAGE PLANS:
table name: default.ice_orc
blocking: true
-PREHOOK: query: alter table ice_orc COMPACT 'major' and wait
+PREHOOK: query: alter table ice_orc COMPACT 'major' and wait where last_name
in ('ln1a', 'ln2a')
PREHOOK: type: ALTERTABLE_COMPACT
PREHOOK: Input: default@ice_orc
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: alter table ice_orc COMPACT 'major' and wait
+POSTHOOK: query: alter table ice_orc COMPACT 'major' and wait where last_name
in ('ln1a', 'ln2a')
POSTHOOK: type: ALTERTABLE_COMPACT
POSTHOOK: Input: default@ice_orc
POSTHOOK: Output: default@ice_orc
@@ -304,12 +248,10 @@ POSTHOOK: query: select * from ice_orc
POSTHOOK: type: QUERY
POSTHOOK: Input: default@ice_orc
#### A masked pattern was here ####
-fn2 ln2a 1 10 100
-fn3 ln3a 1 11 100
-fn4 ln4a 1 11 100
-fn5 ln5a 2 20 100
-fn6 ln6a 2 20 100
-fn7 ln7a 2 21 100
+fn1 ln1a
+fn2 ln2a
+fn3 ln3a
+fn4 ln4a
PREHOOK: query: describe formatted ice_orc
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_orc
@@ -319,14 +261,6 @@ POSTHOOK: Input: default@ice_orc
# col_name data_type comment
first_name string
last_name string
-dept_id bigint
-team_id bigint
-company_id bigint
-
-# Partition Transform Information
-# col_name transform_type
-company_id IDENTITY
-dept_id IDENTITY
# Detailed Table Information
Database: default
@@ -335,24 +269,23 @@ Retention: 0
#### A masked pattern was here ####
Table Type: EXTERNAL_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE
{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"company_id\":\"true\",\"dept_id\":\"true\",\"first_name\":\"true\",\"last_name\":\"true\",\"team_id\":\"true\"}}
+ COLUMN_STATS_ACCURATE
{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"first_name\":\"true\",\"last_name\":\"true\"}}
EXTERNAL TRUE
bucketing_version 2
- current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]}
+ current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"}]}
current-snapshot-id #Masked#
- current-snapshot-summary
{\"added-data-files\":\"1\",\"deleted-data-files\":\"8\",\"removed-position-delete-files\":\"5\",\"removed-delete-files\":\"5\",\"added-records\":\"3\",\"deleted-records\":\"8\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"removed-position-deletes\":\"5\",\"changed-partition-count\":\"5\",\"total-records\":\"6\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"2\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"
[...]
+ current-snapshot-summary
{\"added-data-files\":\"1\",\"deleted-data-files\":\"11\",\"removed-position-delete-files\":\"7\",\"removed-delete-files\":\"7\",\"added-records\":\"4\",\"deleted-records\":\"11\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"removed-position-deletes\":\"7\",\"changed-partition-count\":\"1\",\"total-records\":\"4\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",
[...]
current-snapshot-timestamp-ms #Masked#
- default-partition-spec
{\"spec-id\":1,\"fields\":[{\"name\":\"company_id\",\"transform\":\"identity\",\"source-id\":5,\"field-id\":1000},{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1001}]}
format-version 2
iceberg.orc.files.only true
#### A masked pattern was here ####
- numFiles 2
- numRows 6
+ numFiles 1
+ numRows 4
parquet.compression zstd
#### A masked pattern was here ####
rawDataSize 0
serialization.format 1
- snapshot-count 20
+ snapshot-count 16
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
totalSize #Masked#
@@ -374,6 +307,4 @@ PREHOOK: type: SHOW COMPACTIONS
POSTHOOK: query: show compactions
POSTHOOK: type: SHOW COMPACTIONS
CompactionId Database Table Partition Type State Worker
host Worker Enqueue Time Start Time Duration(ms) HadoopJobId
Error message Initiator host Initiator Pool name TxnId Next
TxnId Commit Time Highest WriteId
-#Masked# default ice_orc company_id=100/dept_id=1 MAJOR
succeeded #Masked# manual default 0 0 0 ---
-#Masked# default ice_orc company_id=100/dept_id=2 MAJOR
succeeded #Masked# manual default 0 0 0 ---
#Masked# default ice_orc --- MAJOR succeeded #Masked#
manual default 0 0 0 ---
diff --git a/itests/src/test/resources/testconfiguration.properties
b/itests/src/test/resources/testconfiguration.properties
index 6768f6b3da5..584a5fdba02 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -422,6 +422,8 @@ iceberg.llap.query.files=\
iceberg.llap.query.compactor.files=\
iceberg_major_compaction_partition_evolution.q,\
iceberg_major_compaction_partition_evolution2.q,\
+ iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q,\
+ iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q,\
iceberg_major_compaction_partitioned.q,\
iceberg_major_compaction_query_metadata.q,\
iceberg_major_compaction_schema_evolution.q,\
@@ -429,6 +431,7 @@ iceberg.llap.query.compactor.files=\
iceberg_major_compaction_single_partition_with_evolution.q,\
iceberg_major_compaction_single_partition_with_evolution2.q,\
iceberg_major_compaction_unpartitioned.q,\
+ iceberg_major_compaction_unpartitioned_w_filter.q,\
iceberg_optimize_table_unpartitioned.q
iceberg.llap.only.query.files=\
diff --git
a/parser/src/java/org/apache/hadoop/hive/ql/parse/AlterClauseParser.g
b/parser/src/java/org/apache/hadoop/hive/ql/parse/AlterClauseParser.g
index 65a35c2e0b7..60283282c9b 100644
--- a/parser/src/java/org/apache/hadoop/hive/ql/parse/AlterClauseParser.g
+++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/AlterClauseParser.g
@@ -463,8 +463,8 @@ compactPool
alterStatementSuffixCompact
@init { gParent.msgs.push("compaction request"); }
@after { gParent.msgs.pop(); }
- : KW_COMPACT compactType=StringLiteral tableImplBuckets? blocking?
compactPool? (KW_WITH KW_OVERWRITE KW_TBLPROPERTIES tableProperties)?
orderByClause?
- -> ^(TOK_ALTERTABLE_COMPACT $compactType tableImplBuckets? blocking?
compactPool? tableProperties? orderByClause?)
+ : KW_COMPACT compactType=StringLiteral tableImplBuckets? blocking?
compactPool? (KW_WITH KW_OVERWRITE KW_TBLPROPERTIES tableProperties)?
orderByClause? whereClause?
+ -> ^(TOK_ALTERTABLE_COMPACT $compactType tableImplBuckets? blocking?
compactPool? tableProperties? orderByClause? whereClause?)
;
alterStatementSuffixSetOwner
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/compact/AlterTableCompactAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/compact/AlterTableCompactAnalyzer.java
index ee7efd5a12b..309d356209d 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/compact/AlterTableCompactAnalyzer.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/compact/AlterTableCompactAnalyzer.java
@@ -18,21 +18,31 @@
package org.apache.hadoop.hive.ql.ddl.table.storage.compact;
+import java.util.HashMap;
import java.util.Map;
import org.antlr.runtime.tree.Tree;
import org.apache.hadoop.hive.common.TableName;
import org.apache.hadoop.hive.metastore.api.CompactionType;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.ddl.DDLDesc.DDLDescWithWriteId;
import org.apache.hadoop.hive.ql.ddl.DDLWork;
import org.apache.hadoop.hive.ql.ddl.DDLSemanticAnalyzerFactory.DDLType;
import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableAnalyzer;
+import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.exec.TaskFactory;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.HiveParser;
+import org.apache.hadoop.hive.ql.parse.RowResolver;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.parse.type.ExprNodeTypeCheck;
+import org.apache.hadoop.hive.ql.parse.type.TypeCheckCtx;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
/**
* Analyzer for compact commands.
@@ -58,6 +68,7 @@ public class AlterTableCompactAnalyzer extends
AbstractAlterTableAnalyzer {
boolean isBlocking = false;
String poolName = null;
String orderBy = null;
+ ExprNodeDesc filterExpr = null;
for (int i = 0; i < command.getChildCount(); i++) {
Tree node = command.getChild(i);
switch (node.getType()) {
@@ -80,13 +91,30 @@ public class AlterTableCompactAnalyzer extends
AbstractAlterTableAnalyzer {
case HiveParser.TOK_ORDERBY:
orderBy =
this.ctx.getTokenRewriteStream().toOriginalString(node.getTokenStartIndex(),
node.getTokenStopIndex());
break;
+ case HiveParser.TOK_WHERE:
+ RowResolver rwsch = new RowResolver();
+ Map<String, String> colTypes = new HashMap<>();
+ try {
+ for (FieldSchema fs : getDb().getTable(tableName).getCols()) {
+ TypeInfo columnType =
TypeInfoUtils.getTypeInfoFromTypeString(fs.getType());
+ rwsch.put(tableName.getTable(), fs.getName(),
+ new ColumnInfo(fs.getName(), columnType, null, true));
+ colTypes.put(fs.getName().toLowerCase(), fs.getType());
+ }
+ } catch (HiveException e) {
+ throw new SemanticException(e);
+ }
+ TypeCheckCtx tcCtx = new TypeCheckCtx(rwsch);
+ ASTNode conds = (ASTNode) node.getChild(0);
+ filterExpr = ExprNodeTypeCheck.genExprNode(conds, tcCtx).get(conds);
+ break;
default:
break;
}
}
AlterTableCompactDesc desc = new AlterTableCompactDesc(tableName,
partitionSpec, type, isBlocking, poolName,
- numberOfBuckets, mapProp, orderBy);
+ numberOfBuckets, mapProp, orderBy, filterExpr);
addInputsOutputsAlterTable(tableName, partitionSpec, desc, desc.getType(),
false);
rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(),
desc)));
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/compact/AlterTableCompactDesc.java
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/compact/AlterTableCompactDesc.java
index eb0d0c2d2ef..f770b5d2aaa 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/compact/AlterTableCompactDesc.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/compact/AlterTableCompactDesc.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hive.ql.ddl.table.AlterTableType;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.Explain;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
/**
* DDL task description for ALTER TABLE ... [PARTITION ... ] COMPACT
'major|minor' [POOL 'poolname'] [WITH OVERWRITE TBLPROPERTIES ...]
@@ -42,11 +43,12 @@ public class AlterTableCompactDesc extends
AbstractAlterTableDesc implements DDL
private final int numberOfBuckets;
private final Map<String, String> properties;
private final String orderByClause;
+ private final ExprNodeDesc filterExpr;
private Long writeId;
public AlterTableCompactDesc(TableName tableName, Map<String, String>
partitionSpec, String compactionType,
- boolean isBlocking, String poolName, int numberOfBuckets, Map<String,
String> properties, String orderByClause)
- throws SemanticException{
+ boolean isBlocking, String poolName, int numberOfBuckets, Map<String,
String> properties, String orderByClause,
+ ExprNodeDesc filterExpr) throws SemanticException{
super(AlterTableType.COMPACT, tableName, partitionSpec, null, false,
false, properties);
this.tableName = tableName.getNotEmptyDbTable();
this.partitionSpec = partitionSpec;
@@ -56,6 +58,7 @@ public class AlterTableCompactDesc extends
AbstractAlterTableDesc implements DDL
this.numberOfBuckets = numberOfBuckets;
this.properties = properties;
this.orderByClause = orderByClause;
+ this.filterExpr = filterExpr;
}
@Explain(displayName = "table name", explainLevels = { Level.USER,
Level.DEFAULT, Level.EXTENDED })
@@ -99,6 +102,10 @@ public class AlterTableCompactDesc extends
AbstractAlterTableDesc implements DDL
return orderByClause;
}
+ public ExprNodeDesc getFilterExpr() {
+ return filterExpr;
+ }
+
@Override
public void setWriteId(long writeId) {
this.writeId = writeId;
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/compact/AlterTableCompactOperation.java
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/compact/AlterTableCompactOperation.java
index 45204d394c5..11afd462638 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/compact/AlterTableCompactOperation.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/compact/AlterTableCompactOperation.java
@@ -63,7 +63,16 @@ public class AlterTableCompactOperation extends
DDLOperation<AlterTableCompactDe
if (!AcidUtils.isTransactionalTable(table) &&
!AcidUtils.isNonNativeAcidTable(table)) {
throw new HiveException(ErrorMsg.NONACID_COMPACTION_NOT_SUPPORTED,
table.getDbName(), table.getTableName());
}
-
+
+ if (desc.getFilterExpr() != null) {
+ if (!DDLUtils.isIcebergTable(table)) {
+ throw new
HiveException(ErrorMsg.NONICEBERG_COMPACTION_WITH_FILTER_NOT_SUPPORTED,
table.getDbName(), table.getTableName());
+ }
+ else if (desc.getPartitionSpec() != null) {
+ throw new
HiveException(ErrorMsg.ICEBERG_COMPACTION_WITH_PART_SPEC_AND_FILTER_NOT_SUPPORTED,
table.getDbName(), table.getTableName());
+ }
+ }
+
if (table.getStorageHandler() != null) {
Optional<ErrorMsg> error =
table.getStorageHandler().isEligibleForCompaction(table,
desc.getPartitionSpec());
if (error.isPresent()) {
@@ -89,6 +98,12 @@ public class AlterTableCompactOperation extends
DDLOperation<AlterTableCompactDe
compactionRequest.setNumberOfBuckets(desc.getNumberOfBuckets());
}
+ // End if filter doesn't match any data on the unpartitioned table
+ if (desc.getPartitionSpec() == null && desc.getFilterExpr() != null
+ && !table.getStorageHandler().hasDataMatchingFilterExpr(table,
desc.getFilterExpr())) {
+ return 0;
+ }
+
//Will directly initiate compaction if an un-partitioned table/a partition
is specified in the request
if (desc.getPartitionSpec() != null || !(table.isPartitioned() ||
(DDLUtils.isIcebergTable(table) &&
table.getStorageHandler().isPartitioned(table)))) {
@@ -108,7 +123,9 @@ public class AlterTableCompactOperation extends
DDLOperation<AlterTableCompactDe
}
// If Iceberg table had partition evolution, it will create compaction
request without partition specification,
// and it will compact all files from old partition specs, besides
compacting partitions of current spec in parallel.
- if (DDLUtils.isIcebergTable(table) &&
table.getStorageHandler().hasUndergonePartitionEvolution(table)) {
+ if (DDLUtils.isIcebergTable(table) &&
table.getStorageHandler().hasUndergonePartitionEvolution(table) &&
+ (desc.getFilterExpr() == null || !table.getStorageHandler()
+ .getPartitionsByExpr(table, desc.getFilterExpr(),
false).isEmpty())) {
compactionRequest.setPartitionname(null);
CompactionResponse compactionResponse =
txnHandler.compact(compactionRequest);
parseCompactionResponse(compactionResponse, table,
compactionRequest.getPartitionname());
@@ -144,7 +161,9 @@ public class AlterTableCompactOperation extends
DDLOperation<AlterTableCompactDe
List<Partition> partitions = new ArrayList<>();
if (desc.getPartitionSpec() == null) {
- if (table.isPartitioned() || (DDLUtils.isIcebergTable(table) &&
table.getStorageHandler().isPartitioned(table))) {
+ if (desc.getFilterExpr() != null && DDLUtils.isIcebergTable(table) &&
table.getStorageHandler().isPartitioned(table)) {
+ partitions = table.getStorageHandler().getPartitionsByExpr(table,
desc.getFilterExpr(), true);
+ } else if (table.isPartitioned() || DDLUtils.isIcebergTable(table) &&
table.getStorageHandler().isPartitioned(table)) {
// Compaction will get initiated for all the potential partitions that
meets the criteria
partitions = context.getDb().getPartitions(table);
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/concatenate/AlterTableConcatenateAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/concatenate/AlterTableConcatenateAnalyzer.java
index 1322ace7e91..7b4921bb045 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/concatenate/AlterTableConcatenateAnalyzer.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/concatenate/AlterTableConcatenateAnalyzer.java
@@ -102,7 +102,7 @@ public class AlterTableConcatenateAnalyzer extends
AbstractAlterTableAnalyzer {
boolean isBlocking = !HiveConf.getBoolVar(conf,
ConfVars.TRANSACTIONAL_CONCATENATE_NOBLOCK, false);
AlterTableCompactDesc desc = new AlterTableCompactDesc(tableName,
partitionSpec, CompactionType.MAJOR.name(), isBlocking,
- poolName, 0, null, null);
+ poolName, 0, null, null, null);
addInputsOutputsAlterTable(tableName, partitionSpec, desc, desc.getType(),
false);
rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(),
desc)));
setAcidDdlDesc(getTable(tableName), desc);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java
index 6a1dbd7ce49..3751004a1a8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java
@@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -83,4 +84,17 @@ public class DummyPartition extends Partition {
return values;
}
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof DummyPartition) {
+ DummyPartition o = (DummyPartition) obj;
+ return Objects.equals(name, o.name) && Objects.equals(partSpec,
o.partSpec);
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(name) + Objects.hashCode(partSpec);
+ }
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java
index 168a91f3c3d..6ff715cdf44 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java
@@ -804,12 +804,32 @@ public interface HiveStorageHandler extends Configurable {
"for a table.");
}
- default List<Partition>
getPartitionsByExpr(org.apache.hadoop.hive.ql.metadata.Table hmsTable,
ExprNodeDesc desc)
+ /**
+ * Returns a list of partitions with the latest partition spec which contain
any files whose content falls under
+ * the provided filter condition.
+ * @param hmsTable {@link org.apache.hadoop.hive.ql.metadata.Table} table
metadata stored in Hive Metastore
+ * @param filter Iceberg filter expression
+ * @return List of Partitions {@link
org.apache.hadoop.hive.ql.metadata.Partition}
+ */
+ default List<Partition>
getPartitionsByExpr(org.apache.hadoop.hive.ql.metadata.Table hmsTable,
ExprNodeDesc filter)
throws SemanticException {
throw new UnsupportedOperationException("Storage handler does not support
getting partitions by expression " +
"for a table.");
}
+ /**
+ * Returns a list of partitions which contain any files whose content falls
under the provided filter condition.
+ * @param hmsTable {@link org.apache.hadoop.hive.ql.metadata.Table} table
metadata stored in Hive Metastore
+ * @param filter Iceberg filter expression
+ * @param latestSpecOnly When true, returns partitions with the latest
partition spec, else with the older specs only.
+ * @return List of Partitions {@link
org.apache.hadoop.hive.ql.metadata.Partition}
+ */
+ default List<Partition>
getPartitionsByExpr(org.apache.hadoop.hive.ql.metadata.Table hmsTable,
+ ExprNodeDesc filter, boolean
latestSpecOnly) throws SemanticException {
+ throw new UnsupportedOperationException("Storage handler does not support
getting partitions " +
+ "by generic expressions");
+ }
+
/**
* Returns partition based on table and partition specification.
* @param table {@link org.apache.hadoop.hive.ql.metadata.Table} table
metadata stored in Hive Metastore
@@ -836,7 +856,7 @@ public interface HiveStorageHandler extends Configurable {
}
/**
- * Returns a list of partitions based on table and partial partition
specification.
+ * Returns a list of partitions with the latest partition spec based on
table and partial partition specification.
* @param table {@link org.apache.hadoop.hive.ql.metadata.Table} table
metadata stored in Hive Metastore
* @param partitionSpec Map of Strings {@link java.util.Map} partition
specification
* @return List of Partitions {@link
org.apache.hadoop.hive.ql.metadata.Partition}
@@ -851,7 +871,7 @@ public interface HiveStorageHandler extends Configurable {
* Returns a list of partitions based on table and partial partition
specification.
* @param table {@link org.apache.hadoop.hive.ql.metadata.Table} table
metadata stored in Hive Metastore
* @param partitionSpec Map of Strings {@link java.util.Map} partition
specification
- * @param latestSpecOnly Specifies if to return only partitions for the
latest partition spec
+ * @param latestSpecOnly When true, returns partitions with the latest
partition spec, else with older specs
* @return List of Partitions {@link
org.apache.hadoop.hive.ql.metadata.Partition}
* @throws SemanticException {@link
org.apache.hadoop.hive.ql.parse.SemanticException}
*/
@@ -869,6 +889,16 @@ public interface HiveStorageHandler extends Configurable {
"undergone partition evolution.");
}
+ /**
+ * Returns true if the table contain any records matching the filter
expression, false otherwise
+ * @param hmsTable {@link org.apache.hadoop.hive.ql.metadata.Table} table
metadata stored in Hive Metastore
+ * @param filter Iceberg filter expression
+ * @return boolean
+ */
+ default boolean
hasDataMatchingFilterExpr(org.apache.hadoop.hive.ql.metadata.Table hmsTable,
ExprNodeDesc filter) {
+ throw new UnsupportedOperationException("Storage handler does not support
checking if filter has any match");
+ }
+
default boolean supportsMergeFiles() {
return false;
}