This is an automated email from the ASF dual-hosted git repository.
amogh-jahagirdar pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new 17d375a0ed Spark: Use merge append by default for streaming writes
(#18332)
17d375a0ed is described below
commit 17d375a0ed3a9213935f59e04404b83c64e982a6
Author: Hongyue/Steve Zhang <[email protected]>
AuthorDate: Mon Oct 5 13:39:22 2026 -0700
Spark: Use merge append by default for streaming writes (#18332)
Spark 3.5, 4.0, 4.1, 4.2: Use merge append by default for streaming writes
Flip the default of the `use-merge-append-for-streaming` write option from
false to true, so streaming appends merge manifests on commit instead of
accumulating a small manifest per commit.
The option was added in 1.12 (#17347, #17403) defaulting to fast append to
give users a release to adjust in case anything relied on the one-manifest-
per-commit behavior. Now that 1.12 has shipped, flip the default for 1.13.
Fast append remains available by setting the option to false.
---
docs/docs/spark-configuration.md | 2 +-
.../spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java | 2 +-
.../spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java | 2 +-
.../spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java | 2 +-
.../spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java | 2 +-
5 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/docs/docs/spark-configuration.md b/docs/docs/spark-configuration.md
index 0b57496072..857f7c40f3 100644
--- a/docs/docs/spark-configuration.md
+++ b/docs/docs/spark-configuration.md
@@ -270,7 +270,7 @@ df.writeTo("catalog.db.table")
| delete-granularity | file | Override this table's delete granularity for
this write |
| shred-variants | false | Overrides this table's write.parquet.shred-variants
for this write |
| variant-inference-buffer-size | 100 | Overrides this table's
write.parquet.variant-inference-buffer-size for this write |
-| use-merge-append-for-streaming | false | Use a merge append instead of a
fast append for streaming appends, so manifests are merged on commit rather
than accumulating as small manifests |
+| use-merge-append-for-streaming | true | Use a merge append instead of a fast
append for streaming appends, so manifests are merged on commit rather than
accumulating as small manifests |
CommitMetadata provides an interface to add custom metadata to a snapshot
summary during a SQL execution, which can be beneficial for purposes such as
auditing or change tracking. If properties start with `snapshot-property.`,
then that prefix will be removed from each property. Here is an example:
diff --git
a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java
b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java
index 09bfd762b7..25f849ed73 100644
---
a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java
+++
b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java
@@ -89,5 +89,5 @@ public class SparkWriteOptions {
// Uses the merge append instead of fast append for streaming appends
public static final String USE_MERGE_APPEND_FOR_STREAMING =
"use-merge-append-for-streaming";
- public static final boolean USE_MERGE_APPEND_FOR_STREAMING_DEFAULT = false;
+ public static final boolean USE_MERGE_APPEND_FOR_STREAMING_DEFAULT = true;
}
diff --git
a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java
b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java
index b7a5597aea..6580e8532a 100644
---
a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java
+++
b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java
@@ -95,5 +95,5 @@ public class SparkWriteOptions {
// Uses the merge append instead of fast append for streaming appends
public static final String USE_MERGE_APPEND_FOR_STREAMING =
"use-merge-append-for-streaming";
- public static final boolean USE_MERGE_APPEND_FOR_STREAMING_DEFAULT = false;
+ public static final boolean USE_MERGE_APPEND_FOR_STREAMING_DEFAULT = true;
}
diff --git
a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java
b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java
index d7a62e2b1e..f56f549529 100644
---
a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java
+++
b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java
@@ -95,5 +95,5 @@ public class SparkWriteOptions {
// Uses the merge append instead of fast append for streaming appends
public static final String USE_MERGE_APPEND_FOR_STREAMING =
"use-merge-append-for-streaming";
- public static final boolean USE_MERGE_APPEND_FOR_STREAMING_DEFAULT = false;
+ public static final boolean USE_MERGE_APPEND_FOR_STREAMING_DEFAULT = true;
}
diff --git
a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java
b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java
index d7a62e2b1e..f56f549529 100644
---
a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java
+++
b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java
@@ -95,5 +95,5 @@ public class SparkWriteOptions {
// Uses the merge append instead of fast append for streaming appends
public static final String USE_MERGE_APPEND_FOR_STREAMING =
"use-merge-append-for-streaming";
- public static final boolean USE_MERGE_APPEND_FOR_STREAMING_DEFAULT = false;
+ public static final boolean USE_MERGE_APPEND_FOR_STREAMING_DEFAULT = true;
}
