This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new b3038e7649c HIVE-28987: [Iceberg] A faulty query predicate can
compromise transaction isolation (#5839)
b3038e7649c is described below
commit b3038e7649c3b1e3727126621e0155e07929c81d
Author: Denys Kuzmenko <[email protected]>
AuthorDate: Wed Jun 18 16:12:42 2025 +0200
HIVE-28987: [Iceberg] A faulty query predicate can compromise transaction
isolation (#5839)
---
.../org/apache/iceberg/mr/InputFormatConfig.java | 1 -
.../iceberg/mr/hive/HiveIcebergInputFormat.java | 3 +-
.../mr/hive/HiveIcebergOutputCommitter.java | 36 +++++----
.../iceberg/mr/hive/HiveIcebergStorageHandler.java | 9 +--
.../hadoop/hive/ql/session/SessionStateUtil.java | 88 ++++++++++++++--------
5 files changed, 82 insertions(+), 55 deletions(-)
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java
index feb9caa8ff7..4898b4b8954 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java
@@ -87,7 +87,6 @@ private InputFormatConfig() {
public static final String CATALOG_WAREHOUSE_TEMPLATE =
"iceberg.catalog.%s.warehouse";
public static final String CATALOG_CLASS_TEMPLATE =
"iceberg.catalog.%s.catalog-impl";
public static final String CATALOG_DEFAULT_CONFIG_PREFIX =
"iceberg.catalog-default.";
- public static final String QUERY_FILTERS = "iceberg.query.filters";
public enum InMemoryDataModel {
HIVE,
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergInputFormat.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergInputFormat.java
index 42522ec0a54..552942a816f 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergInputFormat.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergInputFormat.java
@@ -122,8 +122,7 @@ static Expression getFilterExpr(Configuration conf,
ExprNodeGenericFuncDesc expr
try {
return HiveIcebergFilterFactory.generateFilterExpression(sarg);
} catch (UnsupportedOperationException e) {
- LOG.warn("Unable to create Iceberg filter, proceeding without it (will
be applied by Hive later): {}",
- e.getMessage());
+ LOG.warn("Unable to create Iceberg filter, proceeding without it (will
be applied by Hive later): ", e);
}
}
return null;
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputCommitter.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputCommitter.java
index ef1fc90678d..894790c3d85 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputCommitter.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputCommitter.java
@@ -101,9 +101,11 @@
* Currently independent of the Hive ACID transactions.
*/
public class HiveIcebergOutputCommitter extends OutputCommitter {
+ private static final Logger LOG =
LoggerFactory.getLogger(HiveIcebergOutputCommitter.class);
+
private static final String FOR_COMMIT_EXTENSION = ".forCommit";
+ private static final String CONFLICT_DETECTION_FILTER_MSG = "Conflict
detection Filter Expression: {}";
- private static final Logger LOG =
LoggerFactory.getLogger(HiveIcebergOutputCommitter.class);
private static final HiveIcebergOutputCommitter OUTPUT_COMMITTER = new
HiveIcebergOutputCommitter();
public static HiveIcebergOutputCommitter getInstance() {
@@ -450,26 +452,27 @@ private void commitTable(FileIO io, ExecutorService
executor, OutputTable output
Table table = null;
String branchName = null;
-
Long snapshotId = null;
- Expression filterExpr = Expressions.alwaysTrue();
+ Expression filterExpr = null;
for (JobContext jobContext : jobContexts) {
JobConf conf = jobContext.getJobConf();
+
table = Optional.ofNullable(table).orElseGet(() ->
Catalogs.loadTable(conf, catalogProperties));
branchName = conf.get(InputFormatConfig.OUTPUT_TABLE_SNAPSHOT_REF);
snapshotId = getSnapshotId(outputTable.table, branchName);
- Expression jobContextFilterExpr = (Expression)
SessionStateUtil.getResource(conf, InputFormatConfig.QUERY_FILTERS)
- .orElse(Expressions.alwaysTrue());
- if (!filterExpr.equals(jobContextFilterExpr)) {
- filterExpr = Expressions.and(filterExpr, jobContextFilterExpr);
+ if (filterExpr == null) {
+ filterExpr = SessionStateUtil.getConflictDetectionFilter(conf,
catalogProperties.get(Catalogs.NAME))
+ .map(expr -> HiveIcebergInputFormat.getFilterExpr(conf, expr))
+ .orElse(null);
}
- LOG.debug("Filter Expression :{}", filterExpr);
+
LOG.info("Committing job has started for table: {}, using location: {}",
table, generateJobLocation(outputTable.table.location(), conf,
jobContext.getJobID()));
- int numTasks = SessionStateUtil.getCommitInfo(conf, name).map(info ->
info.get(jobContext.getJobID().toString()))
+ int numTasks = SessionStateUtil.getCommitInfo(conf, name)
+ .map(info -> info.get(jobContext.getJobID().toString()))
.map(SessionStateUtil.CommitInfo::getTaskNum).orElseGet(() -> {
// Fallback logic, if number of tasks are not available in the
config
// If there are reducers, then every reducer will generate a
result file.
@@ -485,6 +488,7 @@ private void commitTable(FileIO io, ExecutorService
executor, OutputTable output
deleteFiles.addAll(writeResults.deleteFiles());
replacedDataFiles.addAll(writeResults.replacedDataFiles());
referencedDataFiles.addAll(writeResults.referencedDataFiles());
+
mergedAndDeletedFiles.addAll(writeResults.mergedAndDeletedFiles());
}
@@ -492,7 +496,7 @@ private void commitTable(FileIO io, ExecutorService
executor, OutputTable output
deleteFiles.removeIf(deleteFile -> mergedAndDeletedFiles.contains(new
Path(String.valueOf(deleteFile.path()))));
FilesForCommit filesForCommit = new FilesForCommit(dataFiles, deleteFiles,
replacedDataFiles, referencedDataFiles,
- Collections.emptySet());
+ Collections.emptySet());
long startTime = System.currentTimeMillis();
if (Operation.IOW != operation) {
@@ -505,7 +509,6 @@ private void commitTable(FileIO io, ExecutorService
executor, OutputTable output
commitWrite(table, branchName, snapshotId, startTime, filesForCommit,
operation, filterExpr);
}
} else {
-
RewritePolicy rewritePolicy =
RewritePolicy.fromString(jobContexts.stream()
.findAny()
.map(x -> x.getJobConf().get(ConfVars.REWRITE_POLICY.varname))
@@ -558,7 +561,10 @@ private void commitWrite(Table table, String branchName,
Long snapshotId, long s
if (snapshotId != null) {
write.validateFromSnapshot(snapshotId);
}
- write.conflictDetectionFilter(filterExpr);
+ if (filterExpr != null) {
+ LOG.debug(CONFLICT_DETECTION_FILTER_MSG, filterExpr);
+ write.conflictDetectionFilter(filterExpr);
+ }
write.validateNoConflictingData();
write.validateNoConflictingDeletes();
commit(write);
@@ -584,8 +590,10 @@ private void commitWrite(Table table, String branchName,
Long snapshotId, long s
if (snapshotId != null) {
write.validateFromSnapshot(snapshotId);
}
- write.conflictDetectionFilter(filterExpr);
-
+ if (filterExpr != null) {
+ LOG.debug(CONFLICT_DETECTION_FILTER_MSG, filterExpr);
+ write.conflictDetectionFilter(filterExpr);
+ }
if (!results.dataFiles().isEmpty()) {
write.validateDeletedFiles();
write.validateNoConflictingDeleteFiles();
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
index 60310176271..dc5ac290462 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
@@ -416,9 +416,9 @@ public DecomposedPredicate decomposePredicate(JobConf
jobConf, Deserializer dese
}
}
predicate.pushedPredicate = (ExprNodeGenericFuncDesc) pushedPredicate;
- Expression filterExpr = HiveIcebergInputFormat.getFilterExpr(conf,
predicate.pushedPredicate);
- if (filterExpr != null) {
- SessionStateUtil.addResource(conf, InputFormatConfig.QUERY_FILTERS,
filterExpr);
+
+ if (pushedPredicate != null) {
+ SessionStateUtil.setConflictDetectionFilter(conf,
jobConf.get(Catalogs.NAME), pushedPredicate);
}
return predicate;
}
@@ -2240,8 +2240,7 @@ public boolean
canPerformMetadataDelete(org.apache.hadoop.hive.ql.metadata.Table
try {
exp = HiveIcebergFilterFactory.generateFilterExpression(sarg);
} catch (UnsupportedOperationException e) {
- LOG.warn("Unable to create Iceberg filter," +
- " continuing without metadata delete: ", e);
+ LOG.warn("Unable to create Iceberg filter, skipping metadata delete: ",
e);
return false;
}
Table table = IcebergTableUtil.getTable(conf, hmsTable.getTTable());
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionStateUtil.java
b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionStateUtil.java
index 0009a54c3a3..84d2aa8f291 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionStateUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionStateUtil.java
@@ -18,23 +18,28 @@
package org.apache.hadoop.hive.ql.session;
-import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
+
+import com.google.common.collect.Maps;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.QueryState;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils;
+import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class SessionStateUtil {
-
private static final Logger LOG =
LoggerFactory.getLogger(SessionStateUtil.class);
+
private static final String COMMIT_INFO_PREFIX = "COMMIT_INFO.";
+ private static final String CONFLICT_DETECTION_FILTER =
"conflictDetectionFilter.";
public static final String DEFAULT_TABLE_LOCATION = "defaultLocation";
private SessionStateUtil() {
-
}
/**
@@ -44,7 +49,8 @@ private SessionStateUtil() {
* could not be found
*/
public static Optional<Object> getResource(Configuration conf, String key) {
- return getQueryState(conf).map(state -> state.getResource(key));
+ return getQueryState(conf)
+ .map(queryState -> queryState.getResource(key));
}
/**
@@ -54,29 +60,24 @@ public static Optional<Object> getResource(Configuration
conf, String key) {
* resource itself could not be found, or the resource is not of type String
*/
public static Optional<String> getProperty(Configuration conf, String key) {
- return getResource(conf, key).filter(o -> o instanceof String).map(o ->
(String) o);
+ return getResource(conf, key).filter(String.class::isInstance)
+ .map(String.class::cast);
}
/**
* @param conf Configuration object used for getting the query state, should
contain the query id
* @param key The resource identifier
* @param resource The resource to save into the QueryState
- * @return whether operation succeeded
*/
- public static boolean addResource(Configuration conf, String key, Object
resource) {
- Optional<QueryState> queryState = getQueryState(conf);
- if (queryState.isPresent()) {
- queryState.get().addResource(key, resource);
- return true;
- } else {
- return false;
- }
+ public static void addResource(Configuration conf, String key, Object
resource) {
+ getQueryState(conf)
+ .ifPresent(queryState -> queryState.addResource(key, resource));
}
public static void addResourceOrThrow(Configuration conf, String key, Object
resource) {
getQueryState(conf)
- .orElseThrow(() -> new IllegalStateException("Query state is
missing; failed to add resource for " + key))
- .addResource(key, resource);
+ .orElseThrow(() -> new IllegalStateException("Query state is missing;
failed to add resource for " + key))
+ .addResource(key, resource);
}
/**
@@ -85,7 +86,8 @@ public static void addResourceOrThrow(Configuration conf,
String key, Object res
* @return the CommitInfo map. Key: jobId, Value: {@link CommitInfo}, or
empty Optional if not present
*/
public static Optional<Map<String, CommitInfo>> getCommitInfo(Configuration
conf, String tableName) {
- return getResource(conf, COMMIT_INFO_PREFIX + tableName).map(o ->
(Map<String, CommitInfo>)o);
+ return getResource(conf, COMMIT_INFO_PREFIX + tableName)
+ .map(obj -> (Map<String, CommitInfo>) obj);
}
/**
@@ -94,30 +96,50 @@ public static Optional<Map<String, CommitInfo>>
getCommitInfo(Configuration conf
* @param jobId The job ID
* @param taskNum The number of successful tasks for the job
* @param additionalProps Any additional properties related to the job commit
- * @return whether the operation succeeded
*/
- public static boolean addCommitInfo(Configuration conf, String tableName,
String jobId, int taskNum,
- Map<String, String> additionalProps) {
+ public static void addCommitInfo(
+ Configuration conf, String tableName, String jobId, int taskNum,
Map<String, String> additionalProps) {
CommitInfo commitInfo = new CommitInfo()
- .withJobID(jobId)
- .withTaskNum(taskNum)
- .withProps(additionalProps);
-
- Optional<Map<String, CommitInfo>> commitInfoMap = getCommitInfo(conf,
tableName);
- if (commitInfoMap.isPresent()) {
- commitInfoMap.get().put(jobId, commitInfo);
- return true;
- }
+ .withJobID(jobId)
+ .withTaskNum(taskNum)
+ .withProps(additionalProps);
+
+ getCommitInfo(conf, tableName)
+ .ifPresentOrElse(commitInfoMap -> commitInfoMap.put(jobId, commitInfo),
+ () -> {
+ Map<String, CommitInfo> newCommitInfoMap = Maps.newHashMap();
+ newCommitInfoMap.put(jobId, commitInfo);
+
+ addResource(conf, COMMIT_INFO_PREFIX + tableName,
newCommitInfoMap);
+ });
+ }
- Map<String, CommitInfo> newCommitInfoMap = new HashMap<>();
- newCommitInfoMap.put(jobId, commitInfo);
+ public static Optional<ExprNodeGenericFuncDesc>
getConflictDetectionFilter(Configuration conf, Object tableName) {
+ return getResource(conf, CONFLICT_DETECTION_FILTER + tableName)
+ .map(ExprNodeGenericFuncDesc.class::cast);
+ }
- return addResource(conf, COMMIT_INFO_PREFIX + tableName, newCommitInfoMap);
+ public static void setConflictDetectionFilter(Configuration conf, String
tableName, ExprNodeDesc filterExpr) {
+ String key = CONFLICT_DETECTION_FILTER + tableName;
+
+ getConflictDetectionFilter(conf, tableName)
+ .ifPresentOrElse(prevFilterExpr -> {
+ if (!prevFilterExpr.isSame(filterExpr)) {
+ ExprNodeDesc disjunction = null;
+ try {
+ disjunction = ExprNodeDescUtils.disjunction(prevFilterExpr,
filterExpr);
+ } catch (UDFArgumentException e) {
+ LOG.warn("Unable to create conflict detection filter,
proceeding without it: ", e);
+ }
+ addResource(conf, key, disjunction);
+ }
+ }, () -> addResource(conf, key, filterExpr));
}
public static Optional<QueryState> getQueryState(Configuration conf) {
- return Optional.ofNullable(SessionState.get()).map(ss ->
ss.getQueryState(HiveConf.getQueryId(conf)));
+ return Optional.ofNullable(SessionState.get())
+ .map(ss -> ss.getQueryState(HiveConf.getQueryId(conf)));
}
/**