This is an automated email from the ASF dual-hosted git repository.

difin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new 981e76512d2 HIVE-29014: Iceberg: Fix wrong row count in Iceberg table 
statistics after upgrade to 1.9.1. (#5867)
981e76512d2 is described below

commit 981e76512d200c8261922b6516a4818b8b5d3c77
Author: Dmitriy Fingerman <[email protected]>
AuthorDate: Tue Jun 17 11:15:26 2025 -0400

    HIVE-29014: Iceberg: Fix wrong row count in Iceberg table statistics after 
upgrade to 1.9.1. (#5867)
---
 ...berg_minor_compaction_partition_evolution.q.out |   4 +-
 .../iceberg_minor_compaction_unpartitioned.q.out   |   4 +-
 iceberg/patched-iceberg-core/pom.xml               |   1 +
 .../org/apache/iceberg/ManifestFilterManager.java  | 546 +++++++++++++++++++++
 4 files changed, 551 insertions(+), 4 deletions(-)

diff --git 
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out
 
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out
index a59d7b7eb9f..2687fd3fb2a 100644
--- 
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out
+++ 
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out
@@ -296,7 +296,7 @@ Table Parameters:
        compactor.threshold.target.size 4000                
        current-schema          
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"}]}
        current-snapshot-id     #Masked#
-       current-snapshot-summary        
{\"added-data-files\":\"1\",\"deleted-data-files\":\"2\",\"added-records\":\"3\",\"deleted-records\":\"4\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"2\",\"total-records\":\"6\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"2\",\"total-delete-files\":\"2\",\"total-position-deletes\":\"2\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"}
+       current-snapshot-summary        
{\"added-data-files\":\"1\",\"deleted-data-files\":\"2\",\"removed-position-delete-files\":\"2\",\"removed-delete-files\":\"2\",\"added-records\":\"3\",\"deleted-records\":\"4\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"removed-position-deletes\":\"2\",\"changed-partition-count\":\"3\",\"total-records\":\"6\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"2\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"
 [...]
        current-snapshot-timestamp-ms   #Masked#       
        default-partition-spec  
{\"spec-id\":1,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000}]}
        format-version          2                   
@@ -304,7 +304,7 @@ Table Parameters:
 #### A masked pattern was here ####
        numFiles                2                   
        numPartitions           2                   
-       numRows                 4                   
+       numRows                 6                   
        parquet.compression     zstd                
 #### A masked pattern was here ####
        rawDataSize             0                   
diff --git 
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_unpartitioned.q.out
 
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_unpartitioned.q.out
index ef9c1f5553e..84d93ccf927 100644
--- 
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_unpartitioned.q.out
+++ 
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_unpartitioned.q.out
@@ -273,13 +273,13 @@ Table Parameters:
        compactor.threshold.target.size 2920                
        current-schema          
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"}]}
        current-snapshot-id     #Masked#
-       current-snapshot-summary        
{\"added-data-files\":\"1\",\"deleted-data-files\":\"2\",\"added-records\":\"7\",\"deleted-records\":\"8\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"7\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"1\",\"total-position-deletes\":\"1\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"}
+       current-snapshot-summary        
{\"added-data-files\":\"1\",\"deleted-data-files\":\"2\",\"removed-position-delete-files\":\"1\",\"removed-delete-files\":\"1\",\"added-records\":\"7\",\"deleted-records\":\"8\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"removed-position-deletes\":\"1\",\"changed-partition-count\":\"1\",\"total-records\":\"7\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"
 [...]
        current-snapshot-timestamp-ms   #Masked#       
        format-version          2                   
        iceberg.orc.files.only  true                
 #### A masked pattern was here ####
        numFiles                1                   
-       numRows                 6                   
+       numRows                 7                   
        parquet.compression     zstd                
 #### A masked pattern was here ####
        rawDataSize             0                   
diff --git a/iceberg/patched-iceberg-core/pom.xml 
b/iceberg/patched-iceberg-core/pom.xml
index 5ba9715e65b..07a01df260d 100644
--- a/iceberg/patched-iceberg-core/pom.xml
+++ b/iceberg/patched-iceberg-core/pom.xml
@@ -97,6 +97,7 @@
                       **/StructLikeMap.class
                       **/StructLikeWrapper.class
                       org.apache.iceberg.PartitionsTable.class
+                      org.apache.iceberg.ManifestFilterManager.class
                   </excludes>
                 </artifactItem>
               </artifactItems>
diff --git 
a/iceberg/patched-iceberg-core/src/main/java/org/apache/iceberg/ManifestFilterManager.java
 
b/iceberg/patched-iceberg-core/src/main/java/org/apache/iceberg/ManifestFilterManager.java
new file mode 100644
index 00000000000..716f8a8a0a8
--- /dev/null
+++ 
b/iceberg/patched-iceberg-core/src/main/java/org/apache/iceberg/ManifestFilterManager.java
@@ -0,0 +1,546 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.expressions.InclusiveMetricsEvaluator;
+import org.apache.iceberg.expressions.ManifestEvaluator;
+import org.apache.iceberg.expressions.ResidualEvaluator;
+import org.apache.iceberg.expressions.StrictMetricsEvaluator;
+import org.apache.iceberg.relocated.com.google.common.base.Joiner;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.CharSequenceSet;
+import org.apache.iceberg.util.ManifestFileUtil;
+import org.apache.iceberg.util.Pair;
+import org.apache.iceberg.util.PartitionSet;
+import org.apache.iceberg.util.StructLikeMap;
+import org.apache.iceberg.util.Tasks;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ * This patch effectively reverts the change introduced in Iceberg version 
1.9.1
+ * (related to #11131, "Core: Optimize MergingSnapshotProducer to use 
referenced manifests to determine if manifest
+ * needs to be rewritten"). The original change caused orphan delete files to 
remain in manifests,
+ * which led to incorrect Iceberg table statistics and inaccurate row counts 
when tables were
+ * queried via Hive. This patch aims to correct the handling of orphan 
positional delete files during manifest
+ * filtering to ensure accurate table state.
+ *
+ */
+
+abstract class ManifestFilterManager<F extends ContentFile<F>> {
+  private static final Logger LOG = 
LoggerFactory.getLogger(ManifestFilterManager.class);
+  private static final Joiner COMMA = Joiner.on(",");
+
+  protected static class DeleteException extends ValidationException {
+    private final String partition;
+
+    private DeleteException(String partition) {
+      super("Operation would delete existing data");
+      this.partition = partition;
+    }
+
+    public String partition() {
+      return partition;
+    }
+  }
+
+  private final Map<Integer, PartitionSpec> specsById;
+  private final PartitionSet deleteFilePartitions;
+  private final Set<F> deleteFiles = newFileSet();
+  private final PartitionSet dropPartitions;
+  private final CharSequenceSet deletePaths = CharSequenceSet.empty();
+  private Expression deleteExpression = Expressions.alwaysFalse();
+  private long minSequenceNumber = 0;
+  private boolean failAnyDelete = false;
+  private boolean failMissingDeletePaths = false;
+  private int duplicateDeleteCount = 0;
+  private boolean caseSensitive = true;
+
+  // cache filtered manifests to avoid extra work when commits fail.
+  private final Map<ManifestFile, ManifestFile> filteredManifests = 
Maps.newConcurrentMap();
+
+  // tracking where files were deleted to validate retries quickly
+  private final Map<ManifestFile, Iterable<F>> filteredManifestToDeletedFiles =
+      Maps.newConcurrentMap();
+
+  private final Supplier<ExecutorService> workerPoolSupplier;
+
+  protected ManifestFilterManager(
+      Map<Integer, PartitionSpec> specsById, Supplier<ExecutorService> 
executorSupplier) {
+    this.specsById = specsById;
+    this.deleteFilePartitions = PartitionSet.create(specsById);
+    this.dropPartitions = PartitionSet.create(specsById);
+    this.workerPoolSupplier = executorSupplier;
+  }
+
+  protected abstract void deleteFile(String location);
+
+  protected abstract ManifestWriter<F> newManifestWriter(PartitionSpec spec);
+
+  protected abstract ManifestReader<F> newManifestReader(ManifestFile 
manifest);
+
+  protected abstract Set<F> newFileSet();
+
+  protected void failAnyDelete() {
+    this.failAnyDelete = true;
+  }
+
+  protected void failMissingDeletePaths() {
+    this.failMissingDeletePaths = true;
+  }
+
+  /**
+   * Add a filter to match files to delete. A file will be deleted if all of 
the rows it contains
+   * match this or any other filter passed to this method.
+   *
+   * @param expr an expression to match rows.
+   */
+  protected void deleteByRowFilter(Expression expr) {
+    Preconditions.checkNotNull(expr, "Cannot delete files using filter: null");
+    invalidateFilteredCache();
+    this.deleteExpression = Expressions.or(deleteExpression, expr);
+  }
+
+  /** Add a partition tuple to drop from the table during the delete phase. */
+  protected void dropPartition(int specId, StructLike partition) {
+    Preconditions.checkNotNull(partition, "Cannot delete files in invalid 
partition: null");
+    invalidateFilteredCache();
+    dropPartitions.add(specId, partition);
+  }
+
+  /**
+   * Set the sequence number used to remove old delete files.
+   *
+   * <p>Delete files with a sequence number older than the given value will be 
removed. By setting
+   * this to the sequence number of the oldest data file in the table, this 
will continuously remove
+   * delete files that are no longer needed because deletes cannot match any 
existing rows in the
+   * table.
+   *
+   * @param sequenceNumber a sequence number used to remove old delete files
+   */
+  protected void dropDeleteFilesOlderThan(long sequenceNumber) {
+    Preconditions.checkArgument(
+        sequenceNumber >= 0, "Invalid minimum data sequence number: %s", 
sequenceNumber);
+    this.minSequenceNumber = sequenceNumber;
+  }
+
+  void caseSensitive(boolean newCaseSensitive) {
+    this.caseSensitive = newCaseSensitive;
+  }
+
+  /** Add a specific path to be deleted in the new snapshot. */
+  void delete(F file) {
+    Preconditions.checkNotNull(file, "Cannot delete file: null");
+    invalidateFilteredCache();
+    deleteFiles.add(file);
+    deleteFilePartitions.add(file.specId(), file.partition());
+  }
+
+  /** Add a specific path to be deleted in the new snapshot. */
+  void delete(CharSequence path) {
+    Preconditions.checkNotNull(path, "Cannot delete file path: null");
+    invalidateFilteredCache();
+    deletePaths.add(path);
+  }
+
+  boolean containsDeletes() {
+    return !deletePaths.isEmpty() || !deleteFiles.isEmpty() || 
deleteExpression != Expressions.alwaysFalse() ||
+        !dropPartitions.isEmpty();
+  }
+
+  /**
+   * Filter deleted files out of a list of manifests.
+   *
+   * @param tableSchema the current table schema
+   * @param manifests a list of manifests to be filtered
+   * @return an array of filtered manifests
+   */
+  List<ManifestFile> filterManifests(Schema tableSchema, List<ManifestFile> 
manifests) {
+    if (manifests == null || manifests.isEmpty()) {
+      validateRequiredDeletes();
+      return ImmutableList.of();
+    }
+
+    ManifestFile[] filtered = new ManifestFile[manifests.size()];
+    // open all of the manifest files in parallel, use index to avoid 
reordering
+    Tasks.range(filtered.length)
+        .stopOnFailure()
+        .throwFailureWhenFinished()
+        .executeWith(workerPoolSupplier.get())
+        .run(
+            index -> {
+              ManifestFile manifest = filterManifest(tableSchema, 
manifests.get(index));
+              filtered[index] = manifest;
+            });
+
+    validateRequiredDeletes(filtered);
+
+    return Arrays.asList(filtered);
+  }
+
+  /**
+   * Creates a snapshot summary builder with the files deleted from the set of 
filtered manifests.
+   *
+   * @param manifests a set of filtered manifests
+   */
+  SnapshotSummary.Builder buildSummary(Iterable<ManifestFile> manifests) {
+    SnapshotSummary.Builder summaryBuilder = SnapshotSummary.builder();
+
+    for (ManifestFile manifest : manifests) {
+      PartitionSpec manifestSpec = specsById.get(manifest.partitionSpecId());
+      Iterable<F> manifestDeletes = 
filteredManifestToDeletedFiles.get(manifest);
+      if (manifestDeletes != null) {
+        for (F file : manifestDeletes) {
+          summaryBuilder.deletedFile(manifestSpec, file);
+        }
+      }
+    }
+
+    summaryBuilder.incrementDuplicateDeletes(duplicateDeleteCount);
+
+    return summaryBuilder;
+  }
+
+  /**
+   * Throws a {@link ValidationException} if any deleted file was not present 
in a filtered
+   * manifest.
+   *
+   * @param manifests a set of filtered manifests
+   */
+  @SuppressWarnings("CollectionUndefinedEquality")
+  private void validateRequiredDeletes(ManifestFile... manifests) {
+    if (failMissingDeletePaths) {
+      Set<F> deletedFiles = deletedFiles(manifests);
+      ValidationException.check(
+          deletedFiles.containsAll(deleteFiles),
+          "Missing required files to delete: %s",
+          COMMA.join(
+              deleteFiles.stream()
+                  .filter(f -> !deletedFiles.contains(f))
+                  .map(ContentFile::location)
+                  .collect(Collectors.toList())));
+
+      CharSequenceSet deletedFilePaths =
+          deletedFiles.stream()
+              .map(ContentFile::location)
+              .collect(Collectors.toCollection(CharSequenceSet::empty));
+
+      ValidationException.check(
+          deletedFilePaths.containsAll(deletePaths),
+          "Missing required files to delete: %s",
+          COMMA.join(Iterables.filter(deletePaths, path -> 
!deletedFilePaths.contains(path))));
+    }
+  }
+
+  private Set<F> deletedFiles(ManifestFile[] manifests) {
+    Set<F> deletedFiles = newFileSet();
+
+    if (manifests != null) {
+      for (ManifestFile manifest : manifests) {
+        Iterable<F> manifestDeletes = 
filteredManifestToDeletedFiles.get(manifest);
+        if (manifestDeletes != null) {
+          for (F file : manifestDeletes) {
+            deletedFiles.add(file);
+          }
+        }
+      }
+    }
+
+    return deletedFiles;
+  }
+
+  /**
+   * Deletes filtered manifests that were created by this class, but are not 
in the committed
+   * manifest set.
+   *
+   * @param committed the set of manifest files that were committed
+   */
+  void cleanUncommitted(Set<ManifestFile> committed) {
+    // iterate over a copy of entries to avoid concurrent modification
+    List<Map.Entry<ManifestFile, ManifestFile>> filterEntries =
+        Lists.newArrayList(filteredManifests.entrySet());
+
+    for (Map.Entry<ManifestFile, ManifestFile> entry : filterEntries) {
+      // remove any new filtered manifests that aren't in the committed list
+      ManifestFile manifest = entry.getKey();
+      ManifestFile filtered = entry.getValue();
+      if (!committed.contains(filtered)) {
+        // only delete if the filtered copy was created
+        if (!manifest.equals(filtered)) {
+          deleteFile(filtered.path());
+        }
+
+        // remove the entry from the cache
+        filteredManifests.remove(manifest);
+      }
+    }
+  }
+
+  private void invalidateFilteredCache() {
+    cleanUncommitted(SnapshotProducer.EMPTY_SET);
+  }
+
+  /**
+   * @return a ManifestReader that is a filtered version of the input manifest.
+   */
+  private ManifestFile filterManifest(Schema tableSchema, ManifestFile 
manifest) {
+    ManifestFile cached = filteredManifests.get(manifest);
+    if (cached != null) {
+      return cached;
+    }
+
+    boolean hasLiveFiles = manifest.hasAddedFiles() || 
manifest.hasExistingFiles();
+    if (!hasLiveFiles || !canContainDeletedFiles(manifest)) {
+      filteredManifests.put(manifest, manifest);
+      return manifest;
+    }
+
+    try (ManifestReader<F> reader = newManifestReader(manifest)) {
+      PartitionSpec spec = reader.spec();
+      PartitionAndMetricsEvaluator evaluator =
+          new PartitionAndMetricsEvaluator(tableSchema, spec, 
deleteExpression);
+
+      // this assumes that the manifest doesn't have files to remove and 
streams through the
+      // manifest without copying data. if a manifest does have a file to 
remove, this will break
+      // out of the loop and move on to filtering the manifest.
+      boolean hasDeletedFiles = manifestHasDeletedFiles(evaluator, reader);
+      if (!hasDeletedFiles) {
+        filteredManifests.put(manifest, manifest);
+        return manifest;
+      }
+
+      return filterManifestWithDeletedFiles(evaluator, manifest, reader);
+
+    } catch (IOException e) {
+      throw new RuntimeIOException(e, "Failed to close manifest: %s", 
manifest);
+    }
+  }
+
+  private boolean canContainDeletedFiles(ManifestFile manifest) {
+    boolean canContainExpressionDeletes;
+    if (deleteExpression != null && deleteExpression != 
Expressions.alwaysFalse()) {
+      ManifestEvaluator manifestEvaluator =
+          ManifestEvaluator.forRowFilter(
+              deleteExpression, specsById.get(manifest.partitionSpecId()), 
caseSensitive);
+      canContainExpressionDeletes = manifestEvaluator.eval(manifest);
+    } else {
+      canContainExpressionDeletes = false;
+    }
+
+    boolean canContainDroppedPartitions;
+    if (!dropPartitions.isEmpty()) {
+      canContainDroppedPartitions =
+          ManifestFileUtil.canContainAny(manifest, dropPartitions, specsById);
+    } else {
+      canContainDroppedPartitions = false;
+    }
+
+    boolean canContainDroppedFiles;
+    if (!deletePaths.isEmpty()) {
+      canContainDroppedFiles = true;
+    } else if (!deleteFiles.isEmpty()) {
+      // because there were no path-only deletes, the set of deleted file 
partitions is valid
+      canContainDroppedFiles =
+          ManifestFileUtil.canContainAny(manifest, deleteFilePartitions, 
specsById);
+    } else {
+      canContainDroppedFiles = false;
+    }
+
+    boolean canContainDropBySeq =
+        manifest.content() == ManifestContent.DELETES && 
manifest.minSequenceNumber() < minSequenceNumber;
+
+    return canContainExpressionDeletes || canContainDroppedPartitions || 
canContainDroppedFiles || canContainDropBySeq;
+  }
+
+  @SuppressWarnings({"CollectionUndefinedEquality", 
"checkstyle:CyclomaticComplexity"})
+  private boolean manifestHasDeletedFiles(
+      PartitionAndMetricsEvaluator evaluator, ManifestReader<F> reader) {
+    boolean isDelete = reader.isDeleteManifestReader();
+
+    for (ManifestEntry<F> entry : reader.liveEntries()) {
+      F file = entry.file();
+      boolean markedForDelete =
+          deletePaths.contains(file.location()) || deleteFiles.contains(file) 
||
+              dropPartitions.contains(file.specId(), file.partition()) ||
+              isDelete && entry.isLive() && entry.dataSequenceNumber() > 0 &&
+                  entry.dataSequenceNumber() < minSequenceNumber;
+
+      if (markedForDelete || evaluator.rowsMightMatch(file)) {
+        boolean allRowsMatch = markedForDelete || 
evaluator.rowsMustMatch(file);
+        ValidationException.check(
+            allRowsMatch || isDelete, // ignore delete files where some 
records may not match the expression
+            "Cannot delete file where some, but not all, rows match filter %s: 
%s",
+            this.deleteExpression,
+            file.location());
+
+        if (allRowsMatch) {
+          if (failAnyDelete) {
+            throw new 
DeleteException(reader.spec().partitionToPath(file.partition()));
+          }
+
+          // as soon as a deleted file is detected, stop scanning
+          return true;
+        }
+      }
+    }
+
+    return false;
+  }
+
+  @SuppressWarnings({"CollectionUndefinedEquality", 
"checkstyle:CyclomaticComplexity"})
+  private ManifestFile filterManifestWithDeletedFiles(
+      PartitionAndMetricsEvaluator evaluator, ManifestFile manifest, 
ManifestReader<F> reader) {
+    boolean isDelete = reader.isDeleteManifestReader();
+    // when this point is reached, there is at least one file that will be 
deleted in the
+    // manifest. produce a copy of the manifest with all deleted files removed.
+    Set<F> deletedFiles = newFileSet();
+
+    try {
+      ManifestWriter<F> writer = newManifestWriter(reader.spec());
+      try {
+        reader
+            .liveEntries()
+            .forEach(
+                entry -> {
+                  F file = entry.file();
+                  boolean markedForDelete =
+                      deletePaths.contains(file.location()) || 
deleteFiles.contains(file) ||
+                          dropPartitions.contains(file.specId(), 
file.partition()) ||
+                          isDelete && entry.isLive() && 
entry.dataSequenceNumber() > 0 &&
+                              entry.dataSequenceNumber() < minSequenceNumber;
+                  if (markedForDelete || evaluator.rowsMightMatch(file)) {
+                    boolean allRowsMatch = markedForDelete || 
evaluator.rowsMustMatch(file);
+                    ValidationException.check(
+                        allRowsMatch || isDelete, // ignore delete files where 
some records may not match
+                        // the expression
+                        "Cannot delete file where some, but not all, rows 
match filter %s: %s",
+                        this.deleteExpression,
+                        file.location());
+
+                    if (allRowsMatch) {
+                      writer.delete(entry);
+
+                      if (deletedFiles.contains(file)) {
+                        LOG.warn(
+                            "Deleting a duplicate path from manifest {}: {}",
+                            manifest.path(),
+                            file.location());
+                        duplicateDeleteCount += 1;
+                      } else {
+                        // only add the file to deletes if it is a new delete
+                        // this keeps the snapshot summary accurate for 
non-duplicate data
+                        deletedFiles.add(file.copyWithoutStats());
+                      }
+                    } else {
+                      writer.existing(entry);
+                    }
+
+                  } else {
+                    writer.existing(entry);
+                  }
+                });
+      } finally {
+        writer.close();
+      }
+
+      // return the filtered manifest as a reader
+      ManifestFile filtered = writer.toManifestFile();
+
+      // update caches
+      filteredManifests.put(manifest, filtered);
+      filteredManifestToDeletedFiles.put(filtered, deletedFiles);
+
+      return filtered;
+
+    } catch (IOException e) {
+      throw new RuntimeIOException(e, "Failed to close manifest writer");
+    }
+  }
+
+  // an evaluator that checks whether rows in a file may/must match a given 
expression
+  // this class first partially evaluates the provided expression using the 
partition tuple
+  // and then checks the remaining part of the expression using metrics 
evaluators
+  private class PartitionAndMetricsEvaluator {
+    private final Schema tableSchema;
+    private final ResidualEvaluator residualEvaluator;
+    private final StructLikeMap<Pair<InclusiveMetricsEvaluator, 
StrictMetricsEvaluator>>
+        metricsEvaluators;
+
+    PartitionAndMetricsEvaluator(Schema tableSchema, PartitionSpec spec, 
Expression expr) {
+      this.tableSchema = tableSchema;
+      this.residualEvaluator = ResidualEvaluator.of(spec, expr, caseSensitive);
+      this.metricsEvaluators = StructLikeMap.create(spec.partitionType());
+    }
+
+    boolean rowsMightMatch(F file) {
+      Pair<InclusiveMetricsEvaluator, StrictMetricsEvaluator> evaluators = 
metricsEvaluators(file);
+      InclusiveMetricsEvaluator inclusiveMetricsEvaluator = evaluators.first();
+      return inclusiveMetricsEvaluator.eval(file);
+    }
+
+    boolean rowsMustMatch(F file) {
+      Pair<InclusiveMetricsEvaluator, StrictMetricsEvaluator> evaluators = 
metricsEvaluators(file);
+      StrictMetricsEvaluator strictMetricsEvaluator = evaluators.second();
+      return strictMetricsEvaluator.eval(file);
+    }
+
+    private Pair<InclusiveMetricsEvaluator, StrictMetricsEvaluator> 
metricsEvaluators(F file) {
+      // ResidualEvaluator removes predicates in the expression using 
strict/inclusive projections
+      // if strict projection returns true -> the pred would return true -> 
replace the pred with
+      // true
+      // if inclusive projection returns false -> the pred would return false 
-> replace the pred
+      // with false
+      // otherwise, keep the original predicate and proceed to other 
predicates in the expression
+      // in other words, ResidualEvaluator returns a part of the expression 
that needs to be
+      // evaluated
+      // for rows in the given partition using metrics
+      PartitionData partition = (PartitionData) file.partition();
+      if (!metricsEvaluators.containsKey(partition)) {
+        Expression residual = residualEvaluator.residualFor(partition);
+        InclusiveMetricsEvaluator inclusive =
+            new InclusiveMetricsEvaluator(tableSchema, residual, 
caseSensitive);
+        StrictMetricsEvaluator strict =
+            new StrictMetricsEvaluator(tableSchema, residual, caseSensitive);
+
+        metricsEvaluators.put(
+            partition.copy(), // The partition may be a re-used container so a 
copy is required
+            Pair.of(inclusive, strict));
+      }
+      return metricsEvaluators.get(partition);
+    }
+  }
+}

Reply via email to