This is an automated email from the ASF dual-hosted git repository.
huaxingao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new c31dd921e1 Core: populate manifest created/replaced/kept count when
commit a snapshot (#15003)
c31dd921e1 is described below
commit c31dd921e1550c643a3ea1fce9d9573cc06e2607
Author: Hongyue/Steve Zhang <[email protected]>
AuthorDate: Thu Feb 19 11:09:41 2026 -0800
Core: populate manifest created/replaced/kept count when commit a snapshot
(#15003)
* Core: populate manifest created/kept count when commit a snapshot
* Also handle replaced-manifest in SnapshotSummary.
* Refactor manifest count handling in SnapshotSummary
update to use AtomicInteger for replaced manifests count in
ManifestFilterManager and ManifestMergeManager.
* Enhance manifest replacement tracking in ManifestMergeManager
Updated the logic for counting replaced manifests during bin-packing to
only decrement the count for manifests from previous snapshots.
* Only increment kept-manifest count if snapshot id is assigned to a
manifest
* Remove redundant tests for manifest metrics in TestCommitReporting and
TestDeleteFiles, and enhance snapshot summary validation in TestTransaction.
* lint
---
.../main/java/org/apache/iceberg/FastAppend.java | 2 +
.../org/apache/iceberg/ManifestFilterManager.java | 24 ++++-
.../org/apache/iceberg/ManifestMergeManager.java | 34 ++++++-
.../apache/iceberg/MergingSnapshotProducer.java | 14 +++
.../java/org/apache/iceberg/SnapshotProducer.java | 28 ++++++
.../org/apache/iceberg/TestCommitReporting.java | 18 ++++
.../java/org/apache/iceberg/TestDeleteFiles.java | 110 ++++++++++++++++++++-
.../java/org/apache/iceberg/TestFastAppend.java | 33 ++++++-
.../java/org/apache/iceberg/TestMergeAppend.java | 43 ++++++--
.../test/java/org/apache/iceberg/TestRowDelta.java | 46 ++++++++-
.../org/apache/iceberg/TestSnapshotSummary.java | 77 ++++++++++-----
.../java/org/apache/iceberg/TestTransaction.java | 12 +++
12 files changed, 397 insertions(+), 44 deletions(-)
diff --git a/core/src/main/java/org/apache/iceberg/FastAppend.java
b/core/src/main/java/org/apache/iceberg/FastAppend.java
index 11459e0ecb..a6caed574a 100644
--- a/core/src/main/java/org/apache/iceberg/FastAppend.java
+++ b/core/src/main/java/org/apache/iceberg/FastAppend.java
@@ -166,6 +166,8 @@ class FastAppend extends SnapshotProducer<AppendFiles>
implements AppendFiles {
manifests.addAll(snapshot.allManifests(ops().io()));
}
+ summaryBuilder.merge(buildManifestCountSummary(manifests, 0));
+
return manifests;
}
diff --git a/core/src/main/java/org/apache/iceberg/ManifestFilterManager.java
b/core/src/main/java/org/apache/iceberg/ManifestFilterManager.java
index 9b5dce4467..7d146d9246 100644
--- a/core/src/main/java/org/apache/iceberg/ManifestFilterManager.java
+++ b/core/src/main/java/org/apache/iceberg/ManifestFilterManager.java
@@ -24,6 +24,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutorService;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import org.apache.iceberg.exceptions.RuntimeIOException;
@@ -74,6 +75,9 @@ abstract class ManifestFilterManager<F extends
ContentFile<F>> {
private final Set<String> manifestsWithDeletes = Sets.newHashSet();
private final PartitionSet dropPartitions;
private final CharSequenceSet deletePaths = CharSequenceSet.empty();
+ // count of manifests that were rewritten with different manifest entry
status during filtering
+ private final AtomicInteger replacedManifestsCount = new AtomicInteger(0);
+
private Expression deleteExpression = Expressions.alwaysFalse();
private long minSequenceNumber = 0;
private boolean failAnyDelete = false;
@@ -313,6 +317,18 @@ abstract class ManifestFilterManager<F extends
ContentFile<F>> {
return deletedFiles;
}
+ /**
+ * Returns the count of manifests that were replaced (rewritten) during
filtering.
+ *
+ * <p>A manifest is considered replaced when a new manifest was created to
replace the original
+ * one (i.e., the original manifest != filtered manifest).
+ *
+ * @return the count of replaced manifests
+ */
+ int replacedManifestsCount() {
+ return replacedManifestsCount.get();
+ }
+
/**
* Deletes filtered manifests that were created by this class, but are not
in the committed
* manifest set.
@@ -329,9 +345,10 @@ abstract class ManifestFilterManager<F extends
ContentFile<F>> {
ManifestFile manifest = entry.getKey();
ManifestFile filtered = entry.getValue();
if (!committed.contains(filtered)) {
- // only delete if the filtered copy was created
+ // only delete if the filtered copy was created (manifest was replaced)
if (!manifest.equals(filtered)) {
deleteFile(filtered.path());
+ replacedManifestsCount.decrementAndGet();
}
// remove the entry from the cache
@@ -342,6 +359,7 @@ abstract class ManifestFilterManager<F extends
ContentFile<F>> {
private void invalidateFilteredCache() {
cleanUncommitted(SnapshotProducer.EMPTY_SET);
+ replacedManifestsCount.set(0);
}
/**
@@ -367,7 +385,9 @@ abstract class ManifestFilterManager<F extends
ContentFile<F>> {
// manifest without copying data. if a manifest does have a file to
remove, this will break
// out of the loop and move on to filtering the manifest.
if (manifestHasDeletedFiles(evaluator, manifest, reader)) {
- return filterManifestWithDeletedFiles(evaluator, manifest, reader);
+ ManifestFile filtered = filterManifestWithDeletedFiles(evaluator,
manifest, reader);
+ replacedManifestsCount.incrementAndGet();
+ return filtered;
} else {
filteredManifests.put(manifest, manifest);
return manifest;
diff --git a/core/src/main/java/org/apache/iceberg/ManifestMergeManager.java
b/core/src/main/java/org/apache/iceberg/ManifestMergeManager.java
index 94eb8a1107..410edcc068 100644
--- a/core/src/main/java/org/apache/iceberg/ManifestMergeManager.java
+++ b/core/src/main/java/org/apache/iceberg/ManifestMergeManager.java
@@ -26,6 +26,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutorService;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Supplier;
import org.apache.iceberg.ManifestEntry.Status;
import org.apache.iceberg.exceptions.RuntimeIOException;
@@ -43,6 +44,8 @@ abstract class ManifestMergeManager<F extends ContentFile<F>>
{
private final int minCountToMerge;
private final boolean mergeEnabled;
+ // track manifests replaced during bin-packing
+ private final AtomicInteger replacedManifestsCount = new AtomicInteger(0);
// cache merge results to reuse when retrying
private final Map<List<ManifestFile>, ManifestFile> mergedManifests =
Maps.newConcurrentMap();
@@ -86,6 +89,18 @@ abstract class ManifestMergeManager<F extends
ContentFile<F>> {
return merged;
}
+ /**
+ * Returns the count of manifests that were replaced (merged) during
bin-packing.
+ *
+ * <p>When multiple manifests are merged into a single manifest, each of the
original manifests is
+ * considered replaced.
+ *
+ * @return the count of replaced manifests
+ */
+ int replacedManifestsCount() {
+ return replacedManifestsCount.get();
+ }
+
void cleanUncommitted(Set<ManifestFile> committed) {
// iterate over a copy of entries to avoid concurrent modification
List<Map.Entry<List<ManifestFile>, ManifestFile>> entries =
@@ -96,8 +111,13 @@ abstract class ManifestMergeManager<F extends
ContentFile<F>> {
ManifestFile merged = entry.getValue();
if (!committed.contains(merged)) {
deleteFile(merged.path());
- // remove the deleted file from the cache
- mergedManifests.remove(entry.getKey());
+ List<ManifestFile> bin = entry.getKey();
+ mergedManifests.remove(bin);
+ for (ManifestFile m : bin) {
+ if (snapshotId() != m.snapshotId()) {
+ replacedManifestsCount.decrementAndGet();
+ }
+ }
}
}
}
@@ -152,7 +172,7 @@ abstract class ManifestMergeManager<F extends
ContentFile<F>> {
// not enough to merge, add all manifest files to the output
list
outputManifests.addAll(bin);
} else {
- // merge the group
+ // merge the bin into a single manifest
outputManifests.add(createManifest(specId, bin));
}
});
@@ -200,8 +220,14 @@ abstract class ManifestMergeManager<F extends
ContentFile<F>> {
ManifestFile manifest = writer.toManifestFile();
- // update the cache
+ // cache the merged manifest to reuse when retrying and track replaced
manifests
mergedManifests.put(bin, manifest);
+ for (ManifestFile m : bin) {
+ // only count manifests from previous snapshots; in-memory manifests are
not replaced
+ if (snapshotId() != m.snapshotId()) {
+ replacedManifestsCount.incrementAndGet();
+ }
+ }
return manifest;
}
diff --git a/core/src/main/java/org/apache/iceberg/MergingSnapshotProducer.java
b/core/src/main/java/org/apache/iceberg/MergingSnapshotProducer.java
index 761f94a830..79dcec3411 100644
--- a/core/src/main/java/org/apache/iceberg/MergingSnapshotProducer.java
+++ b/core/src/main/java/org/apache/iceberg/MergingSnapshotProducer.java
@@ -962,6 +962,20 @@ abstract class MergingSnapshotProducer<ThisT> extends
SnapshotProducer<ThisT> {
Iterables.addAll(manifests,
mergeManager.mergeManifests(unmergedManifests));
Iterables.addAll(manifests,
deleteMergeManager.mergeManifests(unmergedDeleteManifests));
+ // update created/kept/replaced manifest count
+ // replaced manifests come from:
+ // 1. filterManager - manifests rewritten to remove deleted files
+ // 2. deleteFilterManager - delete manifests rewritten to remove deleted
files
+ // 3. mergeManager - data manifests merged via bin-packing
+ // 4. deleteMergeManager - delete manifests merged via bin-packing
+ // Note: rewrittenAppendManifests are NEW manifests (copies), not replaced
ones
+ int replacedManifestsCount =
+ filterManager.replacedManifestsCount()
+ + deleteFilterManager.replacedManifestsCount()
+ + mergeManager.replacedManifestsCount()
+ + deleteMergeManager.replacedManifestsCount();
+ summaryBuilder.merge(buildManifestCountSummary(manifests,
replacedManifestsCount));
+
return manifests;
}
diff --git a/core/src/main/java/org/apache/iceberg/SnapshotProducer.java
b/core/src/main/java/org/apache/iceberg/SnapshotProducer.java
index a8f28855ab..cbae25132d 100644
--- a/core/src/main/java/org/apache/iceberg/SnapshotProducer.java
+++ b/core/src/main/java/org/apache/iceberg/SnapshotProducer.java
@@ -645,6 +645,34 @@ abstract class SnapshotProducer<ThisT> implements
SnapshotUpdate<ThisT> {
return true;
}
+ /**
+ * Builds a snapshot summary with manifest counts.
+ *
+ * @param manifests the list of manifests in the new snapshot
+ * @param replacedManifestsCount the count of manifests that were replaced
(rewritten)
+ * @return a summary builder with manifest count metrics set
+ */
+ protected SnapshotSummary.Builder buildManifestCountSummary(
+ List<ManifestFile> manifests, int replacedManifestsCount) {
+ SnapshotSummary.Builder summaryBuilder = SnapshotSummary.builder();
+ int manifestsCreated = 0;
+ int manifestsKept = 0;
+
+ for (ManifestFile manifest : manifests) {
+ if (snapshotId() == manifest.snapshotId()) {
+ manifestsCreated++;
+ } else if (null != manifest.snapshotId()) {
+ manifestsKept++;
+ }
+ }
+
+ summaryBuilder.set(SnapshotSummary.CREATED_MANIFESTS_COUNT,
String.valueOf(manifestsCreated));
+ summaryBuilder.set(SnapshotSummary.KEPT_MANIFESTS_COUNT,
String.valueOf(manifestsKept));
+ summaryBuilder.set(
+ SnapshotSummary.REPLACED_MANIFESTS_COUNT,
String.valueOf(replacedManifestsCount));
+ return summaryBuilder;
+ }
+
protected List<ManifestFile> writeDataManifests(Collection<DataFile> files,
PartitionSpec spec) {
return writeDataManifests(files, null /* inherit data seq */, spec);
}
diff --git a/core/src/test/java/org/apache/iceberg/TestCommitReporting.java
b/core/src/test/java/org/apache/iceberg/TestCommitReporting.java
index d17348a99c..3d39dc6d38 100644
--- a/core/src/test/java/org/apache/iceberg/TestCommitReporting.java
+++ b/core/src/test/java/org/apache/iceberg/TestCommitReporting.java
@@ -63,6 +63,10 @@ public class TestCommitReporting extends TestBase {
assertThat(metrics.addedFilesSizeInBytes().value()).isEqualTo(20L);
assertThat(metrics.totalFilesSizeInBytes().value()).isEqualTo(20L);
+ assertThat(metrics.manifestsCreated().value()).isEqualTo(1L);
+ assertThat(metrics.manifestsKept().value()).isEqualTo(0L);
+ assertThat(metrics.manifestsReplaced().value()).isEqualTo(0L);
+
// now remove those 2 data files
table.newDelete().deleteFile(FILE_A).deleteFile(FILE_D).commit();
report = reporter.lastCommitReport();
@@ -81,6 +85,11 @@ public class TestCommitReporting extends TestBase {
assertThat(metrics.removedFilesSizeInBytes().value()).isEqualTo(20L);
assertThat(metrics.totalFilesSizeInBytes().value()).isEqualTo(0L);
+
+ // delete rewrites the manifest to mark files as deleted: 1 created, 0
kept, 1 replaced
+ assertThat(metrics.manifestsCreated().value()).isEqualTo(1L);
+ assertThat(metrics.manifestsKept().value()).isEqualTo(0L);
+ assertThat(metrics.manifestsReplaced().value()).isEqualTo(1L);
}
@TestTemplate
@@ -128,6 +137,10 @@ public class TestCommitReporting extends TestBase {
assertThat(metrics.addedFilesSizeInBytes().value()).isEqualTo(totalDeleteContentSize);
assertThat(metrics.totalFilesSizeInBytes().value()).isEqualTo(totalDeleteContentSize);
+ assertThat(metrics.manifestsCreated().value()).isEqualTo(1L);
+ assertThat(metrics.manifestsKept().value()).isEqualTo(0L);
+ assertThat(metrics.manifestsReplaced().value()).isEqualTo(0L);
+
// now remove those 2 positional + 1 equality delete files
table
.newRewrite()
@@ -165,6 +178,11 @@ public class TestCommitReporting extends TestBase {
assertThat(metrics.removedFilesSizeInBytes().value()).isEqualTo(totalDeleteContentSize);
assertThat(metrics.totalFilesSizeInBytes().value()).isEqualTo(0L);
+
+ // rewrite creates 1 manifest (delete manifest rewritten), keeps 0,
replaces 1
+ assertThat(metrics.manifestsCreated().value()).isEqualTo(1L);
+ assertThat(metrics.manifestsKept().value()).isEqualTo(0L);
+ assertThat(metrics.manifestsReplaced().value()).isEqualTo(1L);
}
@TestTemplate
diff --git a/core/src/test/java/org/apache/iceberg/TestDeleteFiles.java
b/core/src/test/java/org/apache/iceberg/TestDeleteFiles.java
index ea0988155b..fc08367d6b 100644
--- a/core/src/test/java/org/apache/iceberg/TestDeleteFiles.java
+++ b/core/src/test/java/org/apache/iceberg/TestDeleteFiles.java
@@ -97,6 +97,10 @@ public class TestDeleteFiles extends TestBase {
table,
table.newAppend().appendFile(FILE_A).appendFile(FILE_B).appendFile(FILE_C),
branch);
Snapshot append = latestSnapshot(readMetadata(), branch);
assertThat(version()).isEqualTo(1);
+ assertThat(append.summary())
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "0");
validateSnapshot(null, append, FILE_A, FILE_B, FILE_C);
commit(table, table.newDelete().deleteFile(FILE_A), branch);
@@ -104,6 +108,11 @@ public class TestDeleteFiles extends TestBase {
assertThat(version()).isEqualTo(2);
assertThat(delete1.allManifests(FILE_IO)).hasSize(1);
+ // delete rewrites manifest: 1 created, 0 kept, 1 replaced
+ assertThat(delete1.summary())
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "1");
validateManifestEntries(
delete1.allManifests(table.io()).get(0),
ids(delete1.snapshotId(), append.snapshotId(), append.snapshotId()),
@@ -113,6 +122,11 @@ public class TestDeleteFiles extends TestBase {
Snapshot delete2 = commit(table, table.newDelete().deleteFile(FILE_B),
branch);
assertThat(version()).isEqualTo(3);
assertThat(delete2.allManifests(FILE_IO)).hasSize(1);
+ // second delete rewrites manifest: 1 created, 0 kept, 1 replaced
+ assertThat(delete2.summary())
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "1");
validateManifestEntries(
delete2.allManifests(FILE_IO).get(0),
ids(delete2.snapshotId(), append.snapshotId()),
@@ -166,6 +180,10 @@ public class TestDeleteFiles extends TestBase {
branch);
assertThat(initialSnapshot.allManifests(FILE_IO)).hasSize(1);
+ assertThat(initialSnapshot.summary())
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "0");
validateManifestEntries(
initialSnapshot.allManifests(FILE_IO).get(0),
ids(initialSnapshot.snapshotId(), initialSnapshot.snapshotId()),
@@ -175,6 +193,10 @@ public class TestDeleteFiles extends TestBase {
// delete the first data file
Snapshot deleteSnapshot = commit(table,
table.newDelete().deleteFile(firstDataFile), branch);
assertThat(deleteSnapshot.allManifests(FILE_IO)).hasSize(1);
+ assertThat(deleteSnapshot.summary())
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "1");
validateManifestEntries(
deleteSnapshot.allManifests(FILE_IO).get(0),
ids(deleteSnapshot.snapshotId(), initialSnapshot.snapshotId()),
@@ -187,6 +209,10 @@ public class TestDeleteFiles extends TestBase {
commit(table,
table.newDelete().deleteFromRowFilter(Expressions.lessThan("id", 7)), branch);
assertThat(finalSnapshot.allManifests(FILE_IO)).hasSize(1);
+ assertThat(finalSnapshot.summary())
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "1");
validateManifestEntries(
finalSnapshot.allManifests(FILE_IO).get(0),
ids(finalSnapshot.snapshotId()),
@@ -207,6 +233,10 @@ public class TestDeleteFiles extends TestBase {
branch);
assertThat(initialSnapshot.allManifests(FILE_IO)).hasSize(1);
+ assertThat(initialSnapshot.summary())
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "0");
validateManifestEntries(
initialSnapshot.allManifests(FILE_IO).get(0),
ids(initialSnapshot.snapshotId(), initialSnapshot.snapshotId()),
@@ -219,6 +249,10 @@ public class TestDeleteFiles extends TestBase {
table,
table.newDelete().deleteFromRowFilter(Expressions.greaterThan("id", 5)),
branch);
assertThat(deleteSnapshot.allManifests(FILE_IO)).hasSize(1);
+ assertThat(deleteSnapshot.summary())
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "1");
validateManifestEntries(
deleteSnapshot.allManifests(FILE_IO).get(0),
ids(initialSnapshot.snapshotId(), deleteSnapshot.snapshotId()),
@@ -239,6 +273,10 @@ public class TestDeleteFiles extends TestBase {
branch);
assertThat(initialSnapshot.allManifests(FILE_IO)).hasSize(1);
+ assertThat(initialSnapshot.summary())
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "0");
validateManifestEntries(
initialSnapshot.allManifests(FILE_IO).get(0),
ids(initialSnapshot.snapshotId(), initialSnapshot.snapshotId()),
@@ -252,6 +290,10 @@ public class TestDeleteFiles extends TestBase {
Snapshot deleteSnapshot =
commit(table, table.newDelete().deleteFromRowFilter(predicate),
branch);
assertThat(deleteSnapshot.allManifests(FILE_IO)).hasSize(1);
+ assertThat(deleteSnapshot.summary())
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "1");
validateManifestEntries(
deleteSnapshot.allManifests(FILE_IO).get(0),
ids(initialSnapshot.snapshotId(), deleteSnapshot.snapshotId()),
@@ -293,7 +335,12 @@ public class TestDeleteFiles extends TestBase {
@TestTemplate
public void testDeleteCaseSensitivity() {
- commit(table,
table.newFastAppend().appendFile(DATA_FILE_BUCKET_0_IDS_0_2), branch);
+ Snapshot append =
+ commit(table,
table.newFastAppend().appendFile(DATA_FILE_BUCKET_0_IDS_0_2), branch);
+ assertThat(append.summary())
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "0");
Expression rowFilter = Expressions.lessThan("iD", 5);
@@ -316,6 +363,10 @@ public class TestDeleteFiles extends TestBase {
table,
table.newDelete().deleteFromRowFilter(rowFilter).caseSensitive(false), branch);
assertThat(deleteSnapshot.allManifests(FILE_IO)).hasSize(1);
+ assertThat(deleteSnapshot.summary())
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "1");
validateManifestEntries(
deleteSnapshot.allManifests(FILE_IO).get(0),
ids(deleteSnapshot.snapshotId()),
@@ -328,13 +379,26 @@ public class TestDeleteFiles extends TestBase {
String testBranch = "testBranch";
table.newAppend().appendFile(FILE_A).appendFile(FILE_B).appendFile(FILE_C).commit();
Snapshot initialSnapshot = table.currentSnapshot();
+ assertThat(initialSnapshot.summary())
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "0");
+
// Delete A on test branch
table.newDelete().deleteFile(FILE_A).toBranch(testBranch).commit();
Snapshot testBranchTip = table.snapshot(testBranch);
+ assertThat(testBranchTip.summary())
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "1");
// Delete B and C on main
table.newDelete().deleteFile(FILE_B).deleteFile(FILE_C).commit();
Snapshot delete2 = table.currentSnapshot();
+ assertThat(delete2.summary())
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "1");
// Verify B and C on testBranch
validateManifestEntries(
@@ -403,9 +467,18 @@ public class TestDeleteFiles extends TestBase {
@TestTemplate
public void testDeleteValidateFileExistence() {
- commit(table, table.newFastAppend().appendFile(FILE_B), branch);
+ Snapshot append = commit(table, table.newFastAppend().appendFile(FILE_B),
branch);
+ assertThat(append.summary())
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "0");
+
Snapshot delete =
commit(table,
table.newDelete().deleteFile(FILE_B).validateFilesExist(), branch);
+ assertThat(delete.summary())
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "1");
validateManifestEntries(
Iterables.getOnlyElement(delete.allManifests(FILE_IO)),
ids(delete.snapshotId()),
@@ -432,15 +505,29 @@ public class TestDeleteFiles extends TestBase {
@TestTemplate
public void testDeleteFilesNoValidation() {
- commit(table, table.newFastAppend().appendFile(FILE_B), branch);
+ Snapshot append = commit(table, table.newFastAppend().appendFile(FILE_B),
branch);
+ assertThat(append.summary())
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "0");
+
Snapshot delete1 = commit(table, table.newDelete().deleteFile(FILE_B),
branch);
+ assertThat(delete1.summary())
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "1");
validateManifestEntries(
Iterables.getOnlyElement(delete1.allManifests(FILE_IO)),
ids(delete1.snapshotId()),
files(FILE_B),
statuses(Status.DELETED));
+ // deleting already deleted file results in no manifest changes
Snapshot delete2 = commit(table, table.newDelete().deleteFile(FILE_B),
branch);
+ assertThat(delete2.summary())
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "0");
assertThat(delete2.allManifests(FILE_IO)).isEmpty();
assertThat(delete2.removedDataFiles(FILE_IO)).isEmpty();
}
@@ -510,6 +597,10 @@ public class TestDeleteFiles extends TestBase {
Snapshot snapshot = latestSnapshot(table, branch);
assertThat(snapshot.sequenceNumber()).isEqualTo(1);
assertThat(table.ops().current().lastSequenceNumber()).isEqualTo(1);
+ assertThat(snapshot.summary())
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "2")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "0");
// deleting by row filter should also remove the orphaned dv1 from delete
manifests
commit(table,
table.newDelete().deleteFromRowFilter(Expressions.lessThan("id", 5)), branch);
@@ -517,6 +608,10 @@ public class TestDeleteFiles extends TestBase {
Snapshot deleteSnap = latestSnapshot(table, branch);
assertThat(deleteSnap.sequenceNumber()).isEqualTo(2);
assertThat(table.ops().current().lastSequenceNumber()).isEqualTo(2);
+ assertThat(deleteSnap.summary())
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "2")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "2");
assertThat(deleteSnap.deleteManifests(table.io())).hasSize(1);
validateDeleteManifest(
@@ -544,6 +639,10 @@ public class TestDeleteFiles extends TestBase {
Snapshot snapshot = latestSnapshot(table, branch);
assertThat(snapshot.sequenceNumber()).isEqualTo(1);
assertThat(table.ops().current().lastSequenceNumber()).isEqualTo(1);
+ assertThat(snapshot.summary())
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "2")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "0");
// deleting by path should also remove the orphaned DV for fileA from
delete manifests
commit(table, table.newDelete().deleteFile(FILE_A.location()), branch);
@@ -551,6 +650,11 @@ public class TestDeleteFiles extends TestBase {
Snapshot deleteSnap = latestSnapshot(table, branch);
assertThat(deleteSnap.sequenceNumber()).isEqualTo(2);
assertThat(table.ops().current().lastSequenceNumber()).isEqualTo(2);
+ // delete rewrites both data and delete manifests
+ assertThat(deleteSnap.summary())
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "2")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "2");
assertThat(deleteSnap.deleteManifests(table.io())).hasSize(1);
validateDeleteManifest(
diff --git a/core/src/test/java/org/apache/iceberg/TestFastAppend.java
b/core/src/test/java/org/apache/iceberg/TestFastAppend.java
index 33153d8454..8f427525e2 100644
--- a/core/src/test/java/org/apache/iceberg/TestFastAppend.java
+++ b/core/src/test/java/org/apache/iceberg/TestFastAppend.java
@@ -170,7 +170,11 @@ public class TestFastAppend extends TestBase {
}
// validate that the metadata summary is correct when using appendManifest
- assertThat(snap.summary()).containsEntry("added-data-files", "2");
+ assertThat(snap.summary())
+ .containsEntry("added-data-files", "2")
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0");
V2Assert.assertEquals("Snapshot sequence number should be 1", 1,
snap.sequenceNumber());
V2Assert.assertEquals(
@@ -214,6 +218,13 @@ public class TestFastAppend extends TestBase {
assertThat(snap.allManifests(FILE_IO).get(1).path()).isEqualTo(manifest.path());
}
+ // validate manifest metrics in the snapshot summary
+ // 2 manifests created: 1 for appendFile (FILE_C, FILE_D) + 1 for
appendManifest
+ assertThat(snap.summary())
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "2")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0");
+
V2Assert.assertEquals("Snapshot sequence number should be 1", 1,
snap.sequenceNumber());
V2Assert.assertEquals(
"Last sequence number should be 1", 1,
readMetadata().lastSequenceNumber());
@@ -459,7 +470,10 @@ public class TestFastAppend extends TestBase {
.containsEntry("added-data-files", "2")
.containsEntry("added-records", "2")
.containsEntry("total-data-files", "2")
- .containsEntry("total-records", "2");
+ .containsEntry("total-records", "2")
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0");
}
@TestTemplate
@@ -551,7 +565,10 @@ public class TestFastAppend extends TestBase {
assertThat(table.currentSnapshot().summary())
.doesNotContainKey(SnapshotSummary.PARTITION_SUMMARY_PROP)
- .containsEntry(SnapshotSummary.CHANGED_PARTITION_COUNT_PROP, "1");
+ .containsEntry(SnapshotSummary.CHANGED_PARTITION_COUNT_PROP, "1")
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0");
}
@TestTemplate
@@ -571,7 +588,10 @@ public class TestFastAppend extends TestBase {
.containsEntry(SnapshotSummary.CHANGED_PARTITION_COUNT_PROP, "1")
.containsEntry(
SnapshotSummary.CHANGED_PARTITION_PREFIX + "data_bucket=0",
- "added-data-files=1,added-records=1,added-files-size=10");
+ "added-data-files=1,added-records=1,added-files-size=10")
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0");
}
@TestTemplate
@@ -588,7 +608,10 @@ public class TestFastAppend extends TestBase {
assertThat(table.currentSnapshot().summary())
.doesNotContainKey(SnapshotSummary.PARTITION_SUMMARY_PROP)
- .containsEntry(SnapshotSummary.CHANGED_PARTITION_COUNT_PROP, "2");
+ .containsEntry(SnapshotSummary.CHANGED_PARTITION_COUNT_PROP, "2")
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0");
}
@TestTemplate
diff --git a/core/src/test/java/org/apache/iceberg/TestMergeAppend.java
b/core/src/test/java/org/apache/iceberg/TestMergeAppend.java
index 0759b0f13a..4863aa37be 100644
--- a/core/src/test/java/org/apache/iceberg/TestMergeAppend.java
+++ b/core/src/test/java/org/apache/iceberg/TestMergeAppend.java
@@ -285,7 +285,11 @@ public class TestMergeAppend extends TestBase {
statuses(Status.ADDED, Status.ADDED));
// validate that the metadata summary is correct when using appendManifest
- assertThat(committedSnapshot.summary()).containsEntry("added-data-files",
"2");
+ assertThat(committedSnapshot.summary())
+ .containsEntry("added-data-files", "2")
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0");
}
@TestTemplate
@@ -514,6 +518,12 @@ public class TestMergeAppend extends TestBase {
ids(commitId1, commitId1),
files(FILE_C, FILE_D),
statuses(Status.ADDED, Status.ADDED));
+ // 3 manifests appended, bin-packing creates 2 bins: [m1] and [m2, m3]
+ // bin 1 kept as-is, bin 2 merged; first snapshot has no prior manifests
+ assertThat(snap1.summary())
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "2")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0");
// produce new manifests as the old ones could have been compacted
manifest = writeManifestWithName("FILE_A_S2", FILE_A);
@@ -559,8 +569,13 @@ public class TestMergeAppend extends TestBase {
files(FILE_A, FILE_C, FILE_D),
statuses(Status.EXISTING, Status.EXISTING, Status.EXISTING));
- // validate that the metadata summary is correct when using appendManifest
- assertThat(snap2.summary()).containsEntry("added-data-files", "3");
+ // 3 new manifests + 2 existing from snap1; bin-packing merges both groups
+ // 2 replaced = existing manifests from snap1 that were merged
+ assertThat(snap2.summary())
+ .containsEntry("added-data-files", "3")
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "3")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "2")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0");
}
@TestTemplate
@@ -781,6 +796,13 @@ public class TestMergeAppend extends TestBase {
V1Assert.assertEquals(
"Table should end with last-sequence-number 0", 0,
readMetadata().lastSequenceNumber());
+ // The delete operation rewrites the original manifest to mark FILE_A as
deleted
+ // This should result in 1 replaced manifest, 1 created manifest, and 0
kept manifests
+ assertThat(deleteSnapshot.summary())
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0");
+
long deleteId = latestSnapshot(table, branch).snapshotId();
assertThat(latestSnapshot(table,
branch).allManifests(table.io())).hasSize(1);
ManifestFile deleteManifest =
deleteSnapshot.allManifests(table.io()).get(0);
@@ -1485,7 +1507,10 @@ public class TestMergeAppend extends TestBase {
assertThat(table.currentSnapshot().summary())
.doesNotContainKey(SnapshotSummary.PARTITION_SUMMARY_PROP)
- .containsEntry(SnapshotSummary.CHANGED_PARTITION_COUNT_PROP, "1");
+ .containsEntry(SnapshotSummary.CHANGED_PARTITION_COUNT_PROP, "1")
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0");
}
@TestTemplate
@@ -1505,7 +1530,10 @@ public class TestMergeAppend extends TestBase {
.containsEntry(SnapshotSummary.CHANGED_PARTITION_COUNT_PROP, "1")
.containsEntry(
SnapshotSummary.CHANGED_PARTITION_PREFIX + "data_bucket=0",
- "added-data-files=1,added-records=1,added-files-size=10");
+ "added-data-files=1,added-records=1,added-files-size=10")
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0");
}
@TestTemplate
@@ -1522,6 +1550,9 @@ public class TestMergeAppend extends TestBase {
assertThat(table.currentSnapshot().summary())
.doesNotContainKey(SnapshotSummary.PARTITION_SUMMARY_PROP)
- .containsEntry(SnapshotSummary.CHANGED_PARTITION_COUNT_PROP, "2");
+ .containsEntry(SnapshotSummary.CHANGED_PARTITION_COUNT_PROP, "2")
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0");
}
}
diff --git a/core/src/test/java/org/apache/iceberg/TestRowDelta.java
b/core/src/test/java/org/apache/iceberg/TestRowDelta.java
index 4eff3a400f..59a73ba202 100644
--- a/core/src/test/java/org/apache/iceberg/TestRowDelta.java
+++ b/core/src/test/java/org/apache/iceberg/TestRowDelta.java
@@ -77,6 +77,10 @@ public class TestRowDelta extends TestBase {
assertThat(snap.sequenceNumber()).isEqualTo(1);
assertThat(snap.operation()).isEqualTo(DataOperations.DELETE);
assertThat(snap.deleteManifests(table.io())).hasSize(1);
+ assertThat(snap.summary())
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "0");
}
@TestTemplate
@@ -88,6 +92,10 @@ public class TestRowDelta extends TestBase {
assertThat(snap.sequenceNumber()).isEqualTo(1);
assertThat(snap.operation()).isEqualTo(DataOperations.APPEND);
assertThat(snap.dataManifests(table.io())).hasSize(1);
+ assertThat(snap.summary())
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "0");
validateManifest(
snap.dataManifests(table.io()).get(0),
@@ -111,6 +119,10 @@ public class TestRowDelta extends TestBase {
.as("Delta commit should use operation 'overwrite'")
.isEqualTo(DataOperations.OVERWRITE);
assertThat(snap.dataManifests(table.io())).hasSize(1);
+ assertThat(snap.summary())
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "2")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "0");
validateManifest(
snap.dataManifests(table.io()).get(0),
@@ -132,7 +144,12 @@ public class TestRowDelta extends TestBase {
@TestTemplate
public void testAddRowsRemoveDataFile() {
- table.newRowDelta().addRows(FILE_A).commit();
+ Snapshot firstSnap = commit(table, table.newRowDelta().addRows(FILE_A),
branch);
+ assertThat(firstSnap.summary())
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "0");
+
SnapshotUpdate<?> rowDelta =
table.newRowDelta().addRows(FILE_B).removeRows(FILE_A);
commit(table, rowDelta, branch);
@@ -143,6 +160,10 @@ public class TestRowDelta extends TestBase {
.as("Delta commit should use operation 'overwrite'")
.isEqualTo(DataOperations.OVERWRITE);
assertThat(snap.dataManifests(table.io())).hasSize(2);
+ assertThat(snap.summary())
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "2")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "1");
validateManifest(
snap.dataManifests(table.io()).get(0),
@@ -1070,6 +1091,10 @@ public class TestRowDelta extends TestBase {
.containsEntry(TOTAL_DELETE_FILES_PROP, "3")
.containsEntry(ADDED_POS_DELETES_PROP, String.valueOf(posDeletesCount))
.containsEntry(TOTAL_POS_DELETES_PROP, String.valueOf(posDeletesCount))
+ // 4 created (1 data + 3 delete), 3 kept (prior data manifests)
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "4")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "3")
.hasEntrySatisfying(
CHANGED_PARTITION_PREFIX + "data_bucket=0",
v -> assertThat(v).contains(ADDED_DELETE_FILES_PROP + "=1"))
@@ -1160,6 +1185,10 @@ public class TestRowDelta extends TestBase {
// 2 appends and 1 row delta where delete files belong to different specs
assertThat(thirdSnapshot.dataManifests(table.io())).hasSize(2);
assertThat(thirdSnapshot.deleteManifests(table.io())).hasSize(2);
+ assertThat(thirdSnapshot.summary())
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "2")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "2")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "0");
// commit two more delete files to the same specs to trigger merging
DeleteFile thirdDeleteFile = newDeletes(firstSnapshotDataFile);
@@ -1603,6 +1632,10 @@ public class TestRowDelta extends TestBase {
RowDelta baseRowDelta =
table.newRowDelta().addRows(dataFile).addDeletes(deleteFile);
Snapshot baseSnapshot = commit(table, baseRowDelta, branch);
assertThat(baseSnapshot.operation()).isEqualTo(DataOperations.OVERWRITE);
+ assertThat(baseSnapshot.summary())
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "2")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "0");
DeleteFile newDeleteFile = newDeletes(dataFile);
RowDelta rowDelta =
@@ -1613,6 +1646,10 @@ public class TestRowDelta extends TestBase {
.validateFromSnapshot(baseSnapshot.snapshotId());
Snapshot snapshot = commit(table, rowDelta, branch);
assertThat(snapshot.operation()).isEqualTo(DataOperations.DELETE);
+ assertThat(snapshot.summary())
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "2")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "1");
List<ManifestFile> dataManifests = snapshot.dataManifests(table.io());
assertThat(dataManifests).hasSize(1);
@@ -1912,8 +1949,13 @@ public class TestRowDelta extends TestBase {
RowDelta rowDelta2 = table.newRowDelta().addDeletes(dv);
Snapshot dvSnapshot = commit(table, rowDelta2, branch);
- // both must be part of the table and merged into one manifest
+ // both delete files merged into one manifest
ManifestFile deleteManifest =
Iterables.getOnlyElement(dvSnapshot.deleteManifests(table.io()));
+ // 1 created (merged), 1 kept (data), 1 replaced (existing delete manifest)
+ assertThat(dvSnapshot.summary())
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "1");
validateDeleteManifest(
deleteManifest,
dataSeqs(3L, 2L),
diff --git a/core/src/test/java/org/apache/iceberg/TestSnapshotSummary.java
b/core/src/test/java/org/apache/iceberg/TestSnapshotSummary.java
index 1eee2d293e..61084b4e67 100644
--- a/core/src/test/java/org/apache/iceberg/TestSnapshotSummary.java
+++ b/core/src/test/java/org/apache/iceberg/TestSnapshotSummary.java
@@ -101,7 +101,7 @@ public class TestSnapshotSummary extends TestBase {
.commit();
assertThat(table.currentSnapshot().summary())
- .hasSize(11)
+ .hasSize(14)
.containsEntry(SnapshotSummary.ADDED_FILES_PROP, "1")
.containsEntry(SnapshotSummary.ADDED_FILE_SIZE_PROP, "10")
.containsEntry(SnapshotSummary.ADDED_RECORDS_PROP, "1")
@@ -111,7 +111,10 @@ public class TestSnapshotSummary extends TestBase {
.containsEntry(SnapshotSummary.TOTAL_EQ_DELETES_PROP, "0")
.containsEntry(SnapshotSummary.TOTAL_POS_DELETES_PROP, "0")
.containsEntry(SnapshotSummary.TOTAL_FILE_SIZE_PROP, "10")
- .containsEntry(SnapshotSummary.TOTAL_RECORDS_PROP, "1");
+ .containsEntry(SnapshotSummary.TOTAL_RECORDS_PROP, "1")
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "0");
}
@TestTemplate
@@ -126,7 +129,7 @@ public class TestSnapshotSummary extends TestBase {
.commit();
assertThat(table.currentSnapshot().summary())
- .hasSize(11)
+ .hasSize(14)
.containsEntry(SnapshotSummary.ADDED_FILES_PROP, "1")
.containsEntry(SnapshotSummary.ADDED_FILE_SIZE_PROP, "10")
.containsEntry(SnapshotSummary.ADDED_RECORDS_PROP, "1")
@@ -136,7 +139,10 @@ public class TestSnapshotSummary extends TestBase {
.containsEntry(SnapshotSummary.TOTAL_EQ_DELETES_PROP, "0")
.containsEntry(SnapshotSummary.TOTAL_POS_DELETES_PROP, "0")
.containsEntry(SnapshotSummary.TOTAL_FILE_SIZE_PROP, "10")
- .containsEntry(SnapshotSummary.TOTAL_RECORDS_PROP, "1");
+ .containsEntry(SnapshotSummary.TOTAL_RECORDS_PROP, "1")
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "0");
}
@TestTemplate
@@ -155,7 +161,7 @@ public class TestSnapshotSummary extends TestBase {
.commit();
assertThat(table.currentSnapshot().summary())
- .hasSize(14)
+ .hasSize(17)
.containsEntry(SnapshotSummary.ADDED_FILES_PROP, "1")
.containsEntry(SnapshotSummary.ADDED_FILE_SIZE_PROP, "10")
.containsEntry(SnapshotSummary.ADDED_RECORDS_PROP, "1")
@@ -168,7 +174,10 @@ public class TestSnapshotSummary extends TestBase {
.containsEntry(SnapshotSummary.TOTAL_EQ_DELETES_PROP, "0")
.containsEntry(SnapshotSummary.TOTAL_POS_DELETES_PROP, "0")
.containsEntry(SnapshotSummary.TOTAL_FILE_SIZE_PROP, "10")
- .containsEntry(SnapshotSummary.TOTAL_RECORDS_PROP, "1");
+ .containsEntry(SnapshotSummary.TOTAL_RECORDS_PROP, "1")
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "2")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "1");
}
@TestTemplate
@@ -187,7 +196,7 @@ public class TestSnapshotSummary extends TestBase {
.commit();
assertThat(table.currentSnapshot().summary())
- .hasSize(11)
+ .hasSize(14)
.containsEntry(SnapshotSummary.CHANGED_PARTITION_COUNT_PROP, "2")
.containsEntry(SnapshotSummary.DELETED_FILES_PROP, "2")
.containsEntry(SnapshotSummary.DELETED_RECORDS_PROP, "2")
@@ -197,7 +206,10 @@ public class TestSnapshotSummary extends TestBase {
.containsEntry(SnapshotSummary.TOTAL_EQ_DELETES_PROP, "0")
.containsEntry(SnapshotSummary.TOTAL_POS_DELETES_PROP, "0")
.containsEntry(SnapshotSummary.TOTAL_FILE_SIZE_PROP, "0")
- .containsEntry(SnapshotSummary.TOTAL_RECORDS_PROP, "0");
+ .containsEntry(SnapshotSummary.TOTAL_RECORDS_PROP, "0")
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "1");
}
@TestTemplate
@@ -212,7 +224,7 @@ public class TestSnapshotSummary extends TestBase {
.commit();
assertThat(table.currentSnapshot().summary())
- .hasSize(12)
+ .hasSize(15)
.containsEntry(SnapshotSummary.ADDED_FILES_PROP, "1")
.containsEntry(SnapshotSummary.ADDED_FILE_SIZE_PROP, "10")
.containsEntry(SnapshotSummary.ADDED_RECORDS_PROP, "1")
@@ -223,7 +235,10 @@ public class TestSnapshotSummary extends TestBase {
.containsEntry(SnapshotSummary.TOTAL_EQ_DELETES_PROP, "0")
.containsEntry(SnapshotSummary.TOTAL_POS_DELETES_PROP, "0")
.containsEntry(SnapshotSummary.TOTAL_FILE_SIZE_PROP, "10")
- .containsEntry(SnapshotSummary.TOTAL_RECORDS_PROP, "1");
+ .containsEntry(SnapshotSummary.TOTAL_RECORDS_PROP, "1")
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "0");
}
@TestTemplate
@@ -238,7 +253,7 @@ public class TestSnapshotSummary extends TestBase {
.commit();
assertThat(table.currentSnapshot().summary())
- .hasSize(11)
+ .hasSize(14)
.containsEntry(SnapshotSummary.ADDED_FILES_PROP, "1")
.containsEntry(SnapshotSummary.ADDED_FILE_SIZE_PROP, "10")
.containsEntry(SnapshotSummary.ADDED_RECORDS_PROP, "1")
@@ -248,7 +263,10 @@ public class TestSnapshotSummary extends TestBase {
.containsEntry(SnapshotSummary.TOTAL_EQ_DELETES_PROP, "0")
.containsEntry(SnapshotSummary.TOTAL_POS_DELETES_PROP, "0")
.containsEntry(SnapshotSummary.TOTAL_FILE_SIZE_PROP, "10")
- .containsEntry(SnapshotSummary.TOTAL_RECORDS_PROP, "1");
+ .containsEntry(SnapshotSummary.TOTAL_RECORDS_PROP, "1")
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "0");
}
@TestTemplate
@@ -267,7 +285,7 @@ public class TestSnapshotSummary extends TestBase {
.commit();
assertThat(table.currentSnapshot().summary())
- .hasSize(14)
+ .hasSize(17)
.containsEntry(SnapshotSummary.ADDED_FILES_PROP, "1")
.containsEntry(SnapshotSummary.ADDED_DELETE_FILES_PROP, "1")
.containsEntry(SnapshotSummary.ADDED_FILE_SIZE_PROP, "20") // size of
data + delete file
@@ -280,7 +298,10 @@ public class TestSnapshotSummary extends TestBase {
.containsEntry(SnapshotSummary.TOTAL_EQ_DELETES_PROP, "0")
.containsEntry(SnapshotSummary.TOTAL_POS_DELETES_PROP, "1")
.containsEntry(SnapshotSummary.TOTAL_FILE_SIZE_PROP, "20")
- .containsEntry(SnapshotSummary.TOTAL_RECORDS_PROP, "1");
+ .containsEntry(SnapshotSummary.TOTAL_RECORDS_PROP, "1")
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "2")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "0");
}
@TestTemplate
@@ -300,7 +321,7 @@ public class TestSnapshotSummary extends TestBase {
.commit();
assertThat(table.currentSnapshot().summary())
- .hasSize(14)
+ .hasSize(17)
.containsEntry(SnapshotSummary.ADDED_FILES_PROP, "1")
.containsEntry(SnapshotSummary.ADDED_FILE_SIZE_PROP, "10")
.containsEntry(SnapshotSummary.ADDED_RECORDS_PROP, "1")
@@ -313,7 +334,10 @@ public class TestSnapshotSummary extends TestBase {
.containsEntry(SnapshotSummary.TOTAL_EQ_DELETES_PROP, "0")
.containsEntry(SnapshotSummary.TOTAL_POS_DELETES_PROP, "0")
.containsEntry(SnapshotSummary.TOTAL_FILE_SIZE_PROP, "10")
- .containsEntry(SnapshotSummary.TOTAL_RECORDS_PROP, "1");
+ .containsEntry(SnapshotSummary.TOTAL_RECORDS_PROP, "1")
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "2")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0");
}
@TestTemplate
@@ -334,7 +358,7 @@ public class TestSnapshotSummary extends TestBase {
.commit();
assertThat(table.currentSnapshot().summary())
- .hasSize(16)
+ .hasSize(19)
.containsEntry(SnapshotSummary.ADDED_DELETE_FILES_PROP, "1")
.containsEntry(SnapshotSummary.ADDED_FILE_SIZE_PROP, "10")
.containsEntry(SnapshotSummary.ADD_POS_DELETE_FILES_PROP, "1")
@@ -349,7 +373,10 @@ public class TestSnapshotSummary extends TestBase {
.containsEntry(SnapshotSummary.TOTAL_EQ_DELETES_PROP, "0")
.containsEntry(SnapshotSummary.TOTAL_POS_DELETES_PROP, "1")
.containsEntry(SnapshotSummary.TOTAL_FILE_SIZE_PROP, "20")
- .containsEntry(SnapshotSummary.TOTAL_RECORDS_PROP, "1");
+ .containsEntry(SnapshotSummary.TOTAL_RECORDS_PROP, "1")
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "2")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "1");
}
@TestTemplate
@@ -368,7 +395,7 @@ public class TestSnapshotSummary extends TestBase {
long totalPosDeletes1 = dv1.recordCount() + dv2.recordCount();
long totalFileSize1 = dv1.contentSizeInBytes() + dv2.contentSizeInBytes();
assertThat(summary1)
- .hasSize(12)
+ .hasSize(15)
.doesNotContainKey(SnapshotSummary.ADD_POS_DELETE_FILES_PROP)
.doesNotContainKey(SnapshotSummary.REMOVED_POS_DELETE_FILES_PROP)
.containsEntry(SnapshotSummary.ADDED_DELETE_FILES_PROP, "1")
@@ -385,7 +412,10 @@ public class TestSnapshotSummary extends TestBase {
.containsEntry(SnapshotSummary.TOTAL_DATA_FILES_PROP, "0")
.containsEntry(SnapshotSummary.TOTAL_EQ_DELETES_PROP, "0")
.containsEntry(SnapshotSummary.TOTAL_RECORDS_PROP, "0")
- .containsEntry(SnapshotSummary.CHANGED_PARTITION_COUNT_PROP, "1");
+ .containsEntry(SnapshotSummary.CHANGED_PARTITION_COUNT_PROP, "1")
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "0");
DeleteFile dv3 = newDV(FILE_A);
table
@@ -404,7 +434,7 @@ public class TestSnapshotSummary extends TestBase {
long totalPosDeletes2 = dv3.recordCount();
long totalFileSize2 = dv3.contentSizeInBytes();
assertThat(summary2)
- .hasSize(16)
+ .hasSize(19)
.doesNotContainKey(SnapshotSummary.ADD_POS_DELETE_FILES_PROP)
.doesNotContainKey(SnapshotSummary.REMOVED_POS_DELETE_FILES_PROP)
.containsEntry(SnapshotSummary.ADDED_DELETE_FILES_PROP, "1")
@@ -421,7 +451,10 @@ public class TestSnapshotSummary extends TestBase {
.containsEntry(SnapshotSummary.TOTAL_DATA_FILES_PROP, "0")
.containsEntry(SnapshotSummary.TOTAL_EQ_DELETES_PROP, "0")
.containsEntry(SnapshotSummary.TOTAL_RECORDS_PROP, "0")
- .containsEntry(SnapshotSummary.CHANGED_PARTITION_COUNT_PROP, "2");
+ .containsEntry(SnapshotSummary.CHANGED_PARTITION_COUNT_PROP, "2")
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "3")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "2");
}
@TestTemplate
diff --git a/core/src/test/java/org/apache/iceberg/TestTransaction.java
b/core/src/test/java/org/apache/iceberg/TestTransaction.java
index 61e0b9ae5a..9ec8c47840 100644
--- a/core/src/test/java/org/apache/iceberg/TestTransaction.java
+++ b/core/src/test/java/org/apache/iceberg/TestTransaction.java
@@ -170,6 +170,18 @@ public class TestTransaction extends TestBase {
ids(appendSnapshot.snapshotId(), appendSnapshot.snapshotId()),
files(FILE_A, FILE_B),
statuses(Status.ADDED, Status.ADDED));
+
+ // validate snapshot summaries for manifest metrics
+ assertThat(appendSnapshot.summary())
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "0");
+
+ // delete rewrites the append manifest
+ assertThat(deleteSnapshot.summary())
+ .containsEntry(SnapshotSummary.CREATED_MANIFESTS_COUNT, "1")
+ .containsEntry(SnapshotSummary.KEPT_MANIFESTS_COUNT, "0")
+ .containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "1");
}
@TestTemplate
