(iceberg) branch main updated: Spark 4.2: Fix rewrite failure on tables created with a sort order (#18493) (#17814)

[russellspitzer]

This is an automated email from the ASF dual-hosted git repository.

RussellSpitzer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git


The following commit(s) were added to refs/heads/main by this push:
     new 1e39a61f26 Spark 4.2: Fix rewrite failure on tables created with a 
sort order (#18493) (#17814)
1e39a61f26 is described below

commit 1e39a61f262730ee9ec23d991d9fe34dfe1737df
Author: raashed <141854182+rshdh...@users.noreply.github.com>
AuthorDate: Wed Oct 14 20:32:05 2026 +0530

    Spark 4.2: Fix rewrite failure on tables created with a sort order (#18493) 
(#17814)
---
 .../org/apache/iceberg/spark/SparkWriteConf.java   |  5 +++-
 .../apache/iceberg/spark/TestSparkWriteConf.java   | 26 ++++++++++++++++++
 .../spark/actions/TestRewriteDataFilesAction.java  | 31 ++++++++++++++++++++++
 3 files changed, 61 insertions(+), 1 deletion(-)

diff --git 
a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java 
b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java
index 1bbffb9366..0b1aaf671b 100644
--- 
a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java
+++ 
b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java
@@ -169,8 +169,11 @@ public class SparkWriteConf {
         
confParser.intConf().option(SparkWriteOptions.OUTPUT_SORT_ORDER_ID).parseOptional();
 
     if (explicitId != null) {
+      // order id 0 is reserved for unsorted order, which is a valid output 
for any table
+      // whether or not the table contains a sort order with that id
       Preconditions.checkArgument(
-          table.sortOrders().containsKey(explicitId),
+          explicitId == SortOrder.unsorted().orderId()
+              || table.sortOrders().containsKey(explicitId),
           "Cannot use output sort order id %s because the table does not 
contain a sort order with that id",
           explicitId);
       return explicitId;
diff --git 
a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/TestSparkWriteConf.java
 
b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/TestSparkWriteConf.java
index 336067c312..dda8be6d7d 100644
--- 
a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/TestSparkWriteConf.java
+++ 
b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/TestSparkWriteConf.java
@@ -55,6 +55,8 @@ import java.util.Map;
 import org.apache.iceberg.DistributionMode;
 import org.apache.iceberg.FileFormat;
 import org.apache.iceberg.ParameterizedTestExtension;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.SortOrder;
 import org.apache.iceberg.Table;
 import org.apache.iceberg.TableProperties;
 import org.apache.iceberg.UpdateProperties;
@@ -665,6 +667,30 @@ public class TestSparkWriteConf extends 
TestBaseWithCatalog {
     
assertThat(writeConfNoOption.outputSortOrderId(SparkWriteRequirements.EMPTY)).isEqualTo(0);
   }
 
+  @TestTemplate
+  void sortOrderWriteConfWithUnsortedId() {
+    Schema schema = validationCatalog.loadTable(tableIdent).schema();
+    validationCatalog.dropTable(tableIdent);
+    Table table =
+        validationCatalog
+            .buildTable(tableIdent, schema)
+            .withSortOrder(SortOrder.builderFor(schema).asc("id").build())
+            .create();
+    
assertThat(table.sortOrders()).doesNotContainKey(SortOrder.unsorted().orderId());
+
+    SparkWriteConf writeConf =
+        new SparkWriteConf(
+            spark,
+            table,
+            new CaseInsensitiveStringMap(
+                ImmutableMap.of(
+                    SparkWriteOptions.OUTPUT_SORT_ORDER_ID,
+                    String.valueOf(SortOrder.unsorted().orderId()))));
+
+    assertThat(writeConf.outputSortOrderId(SparkWriteRequirements.EMPTY))
+        .isEqualTo(SortOrder.unsorted().orderId());
+  }
+
   private void testWriteProperties(List<Map<String, String>> propertiesSuite) {
     withSQLConf(
         propertiesSuite.get(0),
diff --git 
a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java
 
b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java
index 3bd902fafc..3c1830c340 100644
--- 
a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java
+++ 
b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java
@@ -1706,6 +1706,37 @@ public class TestRewriteDataFilesAction extends TestBase 
{
     dataFilesShouldHaveSortOrderIdMatching(table, c3SortOrder);
   }
 
+  @TestTemplate
+  void zOrderWhenTableDeclaredSortOrderAtCreation() throws IOException {
+    Table table =
+        TABLES.create(
+            SCHEMA,
+            PartitionSpec.unpartitioned(),
+            SortOrder.builderFor(SCHEMA).asc("c2").build(),
+            ImmutableMap.of(TableProperties.FORMAT_VERSION, 
String.valueOf(formatVersion)),
+            tableLocation);
+    
assertThat(table.sortOrders()).doesNotContainKey(SortOrder.unsorted().orderId());
+
+    writeRecords(4, 100);
+    table.refresh();
+
+    List<Object[]> originalData = currentData();
+
+    RewriteDataFiles.Result result =
+        basicRewrite(table)
+            .zOrder("c2", "c3")
+            .option(SizeBasedFileRewritePlanner.REWRITE_ALL, "true")
+            .execute();
+
+    assertThat(result.rewriteResults()).as("Should have 1 
fileGroup").hasSize(1);
+
+    table.refresh();
+
+    assertEquals("We shouldn't have changed the data", originalData, 
currentData());
+    dataFilesShouldHaveSortOrderIdMatching(table, SortOrder.unsorted());
+    shouldHaveACleanCache(table);
+  }
+
   @TestTemplate
   public void testAutoSortShuffleOutput() throws IOException {
     Table table = createTable(20, LARGE_SCALE);