This is an automated email from the ASF dual-hosted git repository.

dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new b3038e7649c HIVE-28987: [Iceberg] A faulty query predicate can 
compromise transaction isolation (#5839)
b3038e7649c is described below

commit b3038e7649c3b1e3727126621e0155e07929c81d
Author: Denys Kuzmenko <[email protected]>
AuthorDate: Wed Jun 18 16:12:42 2025 +0200

    HIVE-28987: [Iceberg] A faulty query predicate can compromise transaction 
isolation (#5839)
---
 .../org/apache/iceberg/mr/InputFormatConfig.java   |  1 -
 .../iceberg/mr/hive/HiveIcebergInputFormat.java    |  3 +-
 .../mr/hive/HiveIcebergOutputCommitter.java        | 36 +++++----
 .../iceberg/mr/hive/HiveIcebergStorageHandler.java |  9 +--
 .../hadoop/hive/ql/session/SessionStateUtil.java   | 88 ++++++++++++++--------
 5 files changed, 82 insertions(+), 55 deletions(-)

diff --git 
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java
 
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java
index feb9caa8ff7..4898b4b8954 100644
--- 
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java
+++ 
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java
@@ -87,7 +87,6 @@ private InputFormatConfig() {
   public static final String CATALOG_WAREHOUSE_TEMPLATE = 
"iceberg.catalog.%s.warehouse";
   public static final String CATALOG_CLASS_TEMPLATE = 
"iceberg.catalog.%s.catalog-impl";
   public static final String CATALOG_DEFAULT_CONFIG_PREFIX = 
"iceberg.catalog-default.";
-  public static final String QUERY_FILTERS = "iceberg.query.filters";
 
   public enum InMemoryDataModel {
     HIVE,
diff --git 
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergInputFormat.java
 
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergInputFormat.java
index 42522ec0a54..552942a816f 100644
--- 
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergInputFormat.java
+++ 
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergInputFormat.java
@@ -122,8 +122,7 @@ static Expression getFilterExpr(Configuration conf, 
ExprNodeGenericFuncDesc expr
       try {
         return HiveIcebergFilterFactory.generateFilterExpression(sarg);
       } catch (UnsupportedOperationException e) {
-        LOG.warn("Unable to create Iceberg filter, proceeding without it (will 
be applied by Hive later): {}",
-            e.getMessage());
+        LOG.warn("Unable to create Iceberg filter, proceeding without it (will 
be applied by Hive later): ", e);
       }
     }
     return null;
diff --git 
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputCommitter.java
 
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputCommitter.java
index ef1fc90678d..894790c3d85 100644
--- 
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputCommitter.java
+++ 
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputCommitter.java
@@ -101,9 +101,11 @@
  * Currently independent of the Hive ACID transactions.
  */
 public class HiveIcebergOutputCommitter extends OutputCommitter {
+  private static final Logger LOG = 
LoggerFactory.getLogger(HiveIcebergOutputCommitter.class);
+
   private static final String FOR_COMMIT_EXTENSION = ".forCommit";
+  private static final String CONFLICT_DETECTION_FILTER_MSG = "Conflict 
detection Filter Expression: {}";
 
-  private static final Logger LOG = 
LoggerFactory.getLogger(HiveIcebergOutputCommitter.class);
   private static final HiveIcebergOutputCommitter OUTPUT_COMMITTER = new 
HiveIcebergOutputCommitter();
 
   public static HiveIcebergOutputCommitter getInstance() {
@@ -450,26 +452,27 @@ private void commitTable(FileIO io, ExecutorService 
executor, OutputTable output
 
     Table table = null;
     String branchName = null;
-
     Long snapshotId = null;
-    Expression filterExpr = Expressions.alwaysTrue();
+    Expression filterExpr = null;
 
     for (JobContext jobContext : jobContexts) {
       JobConf conf = jobContext.getJobConf();
+
       table = Optional.ofNullable(table).orElseGet(() -> 
Catalogs.loadTable(conf, catalogProperties));
       branchName = conf.get(InputFormatConfig.OUTPUT_TABLE_SNAPSHOT_REF);
       snapshotId = getSnapshotId(outputTable.table, branchName);
 
-      Expression jobContextFilterExpr = (Expression) 
SessionStateUtil.getResource(conf, InputFormatConfig.QUERY_FILTERS)
-          .orElse(Expressions.alwaysTrue());
-      if (!filterExpr.equals(jobContextFilterExpr)) {
-        filterExpr = Expressions.and(filterExpr, jobContextFilterExpr);
+      if (filterExpr == null) {
+        filterExpr = SessionStateUtil.getConflictDetectionFilter(conf, 
catalogProperties.get(Catalogs.NAME))
+            .map(expr -> HiveIcebergInputFormat.getFilterExpr(conf, expr))
+            .orElse(null);
       }
-      LOG.debug("Filter Expression :{}", filterExpr);
+
       LOG.info("Committing job has started for table: {}, using location: {}",
           table, generateJobLocation(outputTable.table.location(), conf, 
jobContext.getJobID()));
 
-      int numTasks = SessionStateUtil.getCommitInfo(conf, name).map(info -> 
info.get(jobContext.getJobID().toString()))
+      int numTasks = SessionStateUtil.getCommitInfo(conf, name)
+          .map(info -> info.get(jobContext.getJobID().toString()))
           .map(SessionStateUtil.CommitInfo::getTaskNum).orElseGet(() -> {
             // Fallback logic, if number of tasks are not available in the 
config
             // If there are reducers, then every reducer will generate a 
result file.
@@ -485,6 +488,7 @@ private void commitTable(FileIO io, ExecutorService 
executor, OutputTable output
       deleteFiles.addAll(writeResults.deleteFiles());
       replacedDataFiles.addAll(writeResults.replacedDataFiles());
       referencedDataFiles.addAll(writeResults.referencedDataFiles());
+
       mergedAndDeletedFiles.addAll(writeResults.mergedAndDeletedFiles());
     }
 
@@ -492,7 +496,7 @@ private void commitTable(FileIO io, ExecutorService 
executor, OutputTable output
     deleteFiles.removeIf(deleteFile -> mergedAndDeletedFiles.contains(new 
Path(String.valueOf(deleteFile.path()))));
 
     FilesForCommit filesForCommit = new FilesForCommit(dataFiles, deleteFiles, 
replacedDataFiles, referencedDataFiles,
-            Collections.emptySet());
+        Collections.emptySet());
     long startTime = System.currentTimeMillis();
 
     if (Operation.IOW != operation) {
@@ -505,7 +509,6 @@ private void commitTable(FileIO io, ExecutorService 
executor, OutputTable output
         commitWrite(table, branchName, snapshotId, startTime, filesForCommit, 
operation, filterExpr);
       }
     } else {
-
       RewritePolicy rewritePolicy = 
RewritePolicy.fromString(jobContexts.stream()
           .findAny()
           .map(x -> x.getJobConf().get(ConfVars.REWRITE_POLICY.varname))
@@ -558,7 +561,10 @@ private void commitWrite(Table table, String branchName, 
Long snapshotId, long s
       if (snapshotId != null) {
         write.validateFromSnapshot(snapshotId);
       }
-      write.conflictDetectionFilter(filterExpr);
+      if (filterExpr != null) {
+        LOG.debug(CONFLICT_DETECTION_FILTER_MSG, filterExpr);
+        write.conflictDetectionFilter(filterExpr);
+      }
       write.validateNoConflictingData();
       write.validateNoConflictingDeletes();
       commit(write);
@@ -584,8 +590,10 @@ private void commitWrite(Table table, String branchName, 
Long snapshotId, long s
       if (snapshotId != null) {
         write.validateFromSnapshot(snapshotId);
       }
-      write.conflictDetectionFilter(filterExpr);
-
+      if (filterExpr != null) {
+        LOG.debug(CONFLICT_DETECTION_FILTER_MSG, filterExpr);
+        write.conflictDetectionFilter(filterExpr);
+      }
       if (!results.dataFiles().isEmpty()) {
         write.validateDeletedFiles();
         write.validateNoConflictingDeleteFiles();
diff --git 
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
 
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
index 60310176271..dc5ac290462 100644
--- 
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
+++ 
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
@@ -416,9 +416,9 @@ public DecomposedPredicate decomposePredicate(JobConf 
jobConf, Deserializer dese
       }
     }
     predicate.pushedPredicate = (ExprNodeGenericFuncDesc) pushedPredicate;
-    Expression filterExpr = HiveIcebergInputFormat.getFilterExpr(conf, 
predicate.pushedPredicate);
-    if (filterExpr != null) {
-      SessionStateUtil.addResource(conf, InputFormatConfig.QUERY_FILTERS, 
filterExpr);
+
+    if (pushedPredicate != null) {
+      SessionStateUtil.setConflictDetectionFilter(conf, 
jobConf.get(Catalogs.NAME), pushedPredicate);
     }
     return predicate;
   }
@@ -2240,8 +2240,7 @@ public boolean 
canPerformMetadataDelete(org.apache.hadoop.hive.ql.metadata.Table
     try {
       exp = HiveIcebergFilterFactory.generateFilterExpression(sarg);
     } catch (UnsupportedOperationException e) {
-      LOG.warn("Unable to create Iceberg filter," +
-              " continuing without metadata delete: ", e);
+      LOG.warn("Unable to create Iceberg filter, skipping metadata delete: ", 
e);
       return false;
     }
     Table table = IcebergTableUtil.getTable(conf, hmsTable.getTTable());
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionStateUtil.java 
b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionStateUtil.java
index 0009a54c3a3..84d2aa8f291 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionStateUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionStateUtil.java
@@ -18,23 +18,28 @@
 
 package org.apache.hadoop.hive.ql.session;
 
-import java.util.HashMap;
 import java.util.Map;
 import java.util.Optional;
+
+import com.google.common.collect.Maps;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.QueryState;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils;
+import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 public class SessionStateUtil {
-
   private static final Logger LOG = 
LoggerFactory.getLogger(SessionStateUtil.class);
+
   private static final String COMMIT_INFO_PREFIX = "COMMIT_INFO.";
+  private static final String CONFLICT_DETECTION_FILTER = 
"conflictDetectionFilter.";
   public static final String DEFAULT_TABLE_LOCATION = "defaultLocation";
 
   private SessionStateUtil() {
-
   }
 
   /**
@@ -44,7 +49,8 @@ private SessionStateUtil() {
    * could not be found
    */
   public static Optional<Object> getResource(Configuration conf, String key) {
-    return getQueryState(conf).map(state -> state.getResource(key));
+    return getQueryState(conf)
+        .map(queryState -> queryState.getResource(key));
   }
 
   /**
@@ -54,29 +60,24 @@ public static Optional<Object> getResource(Configuration 
conf, String key) {
    * resource itself could not be found, or the resource is not of type String
    */
   public static Optional<String> getProperty(Configuration conf, String key) {
-    return getResource(conf, key).filter(o -> o instanceof String).map(o -> 
(String) o);
+    return getResource(conf, key).filter(String.class::isInstance)
+        .map(String.class::cast);
   }
 
   /**
    * @param conf Configuration object used for getting the query state, should 
contain the query id
    * @param key The resource identifier
    * @param resource The resource to save into the QueryState
-   * @return whether operation succeeded
    */
-  public static boolean addResource(Configuration conf, String key, Object 
resource) {
-    Optional<QueryState> queryState = getQueryState(conf);
-    if (queryState.isPresent()) {
-      queryState.get().addResource(key, resource);
-      return true;
-    } else {
-      return false;
-    }
+  public static void addResource(Configuration conf, String key, Object 
resource) {
+    getQueryState(conf)
+        .ifPresent(queryState -> queryState.addResource(key, resource));
   }
 
   public static void addResourceOrThrow(Configuration conf, String key, Object 
resource) {
     getQueryState(conf)
-            .orElseThrow(() -> new IllegalStateException("Query state is 
missing; failed to add resource for " + key))
-            .addResource(key, resource);
+        .orElseThrow(() -> new IllegalStateException("Query state is missing; 
failed to add resource for " + key))
+        .addResource(key, resource);
   }
 
   /**
@@ -85,7 +86,8 @@ public static void addResourceOrThrow(Configuration conf, 
String key, Object res
    * @return the CommitInfo map. Key: jobId, Value: {@link CommitInfo}, or 
empty Optional if not present
    */
   public static Optional<Map<String, CommitInfo>> getCommitInfo(Configuration 
conf, String tableName) {
-    return getResource(conf, COMMIT_INFO_PREFIX + tableName).map(o -> 
(Map<String, CommitInfo>)o);
+    return getResource(conf, COMMIT_INFO_PREFIX + tableName)
+        .map(obj -> (Map<String, CommitInfo>) obj);
   }
 
   /**
@@ -94,30 +96,50 @@ public static Optional<Map<String, CommitInfo>> 
getCommitInfo(Configuration conf
    * @param jobId The job ID
    * @param taskNum The number of successful tasks for the job
    * @param additionalProps Any additional properties related to the job commit
-   * @return whether the operation succeeded
    */
-  public static boolean addCommitInfo(Configuration conf, String tableName, 
String jobId, int taskNum,
-                                         Map<String, String> additionalProps) {
+  public static void addCommitInfo(
+      Configuration conf, String tableName, String jobId, int taskNum, 
Map<String, String> additionalProps) {
 
     CommitInfo commitInfo = new CommitInfo()
-            .withJobID(jobId)
-            .withTaskNum(taskNum)
-            .withProps(additionalProps);
-
-    Optional<Map<String, CommitInfo>> commitInfoMap = getCommitInfo(conf, 
tableName);
-    if (commitInfoMap.isPresent()) {
-      commitInfoMap.get().put(jobId, commitInfo);
-      return true;
-    }
+        .withJobID(jobId)
+        .withTaskNum(taskNum)
+        .withProps(additionalProps);
+
+    getCommitInfo(conf, tableName)
+        .ifPresentOrElse(commitInfoMap -> commitInfoMap.put(jobId, commitInfo),
+            () -> {
+              Map<String, CommitInfo> newCommitInfoMap = Maps.newHashMap();
+              newCommitInfoMap.put(jobId, commitInfo);
+
+              addResource(conf, COMMIT_INFO_PREFIX + tableName, 
newCommitInfoMap);
+            });
+  }
 
-    Map<String, CommitInfo> newCommitInfoMap = new HashMap<>();
-    newCommitInfoMap.put(jobId, commitInfo);
+  public static Optional<ExprNodeGenericFuncDesc> 
getConflictDetectionFilter(Configuration conf, Object tableName) {
+    return getResource(conf, CONFLICT_DETECTION_FILTER + tableName)
+        .map(ExprNodeGenericFuncDesc.class::cast);
+  }
 
-    return addResource(conf, COMMIT_INFO_PREFIX + tableName, newCommitInfoMap);
+  public static void setConflictDetectionFilter(Configuration conf, String 
tableName, ExprNodeDesc filterExpr) {
+    String key = CONFLICT_DETECTION_FILTER + tableName;
+
+    getConflictDetectionFilter(conf, tableName)
+        .ifPresentOrElse(prevFilterExpr -> {
+            if (!prevFilterExpr.isSame(filterExpr)) {
+              ExprNodeDesc disjunction = null;
+              try {
+                disjunction = ExprNodeDescUtils.disjunction(prevFilterExpr, 
filterExpr);
+              } catch (UDFArgumentException e) {
+                LOG.warn("Unable to create conflict detection filter, 
proceeding without it: ", e);
+              }
+              addResource(conf, key, disjunction);
+            }
+        }, () -> addResource(conf, key, filterExpr));
   }
 
   public static Optional<QueryState> getQueryState(Configuration conf) {
-    return Optional.ofNullable(SessionState.get()).map(ss -> 
ss.getQueryState(HiveConf.getQueryId(conf)));
+    return Optional.ofNullable(SessionState.get())
+        .map(ss -> ss.getQueryState(HiveConf.getQueryId(conf)));
   }
 
   /**

Reply via email to