This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 69b32d3c56 [iceberg] Preserve table properties during Iceberg to
Paimon migration (#6388)
69b32d3c56 is described below
commit 69b32d3c5631061a531a6491666bb224188a0f6c
Author: Jiajia Li <[email protected]>
AuthorDate: Mon Oct 13 21:00:29 2025 +0800
[iceberg] Preserve table properties during Iceberg to Paimon migration
(#6388)
---
.../apache/paimon/iceberg/metadata/IcebergRef.java | 2 +-
.../paimon/iceberg/migrate/IcebergMigrator.java | 12 ++++---
.../paimon/iceberg/migrate/IcebergMigrateTest.java | 41 ++++++++++++++++++++++
3 files changed, 50 insertions(+), 5 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergRef.java
b/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergRef.java
index c808fcd7b7..c0a1cb227f 100644
---
a/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergRef.java
+++
b/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergRef.java
@@ -30,7 +30,7 @@ import java.util.Objects;
/**
* Iceberg's ref metadata.
*
- * <p>See <a href="https://iceberg.apache.org/spec/#refs";>Iceberg spec</a>.
+ * <p>See <a
href="https://iceberg.apache.org/spec/#snapshot-references";>Iceberg spec</a>.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public class IcebergRef {
diff --git
a/paimon-core/src/main/java/org/apache/paimon/iceberg/migrate/IcebergMigrator.java
b/paimon-core/src/main/java/org/apache/paimon/iceberg/migrate/IcebergMigrator.java
index 0132e14276..0390d241b4 100644
---
a/paimon-core/src/main/java/org/apache/paimon/iceberg/migrate/IcebergMigrator.java
+++
b/paimon-core/src/main/java/org/apache/paimon/iceberg/migrate/IcebergMigrator.java
@@ -291,6 +291,10 @@ public class IcebergMigrator implements Migrator {
}
private List<TableSchema> icebergSchemasToPaimonSchemas(IcebergMetadata
icebergMetadata) {
+ Map<String, String> options =
+ icebergMetadata.properties() == null
+ ? Collections.emptyMap()
+ : icebergMetadata.properties();
return icebergMetadata.schemas().stream()
.map(
icebergSchema -> {
@@ -299,12 +303,13 @@ public class IcebergMigrator implements Migrator {
icebergSchema.schemaId());
return TableSchema.create(
icebergSchema.schemaId(),
-
icebergSchemaToPaimonSchema(icebergSchema));
+ icebergSchemaToPaimonSchema(icebergSchema,
options));
})
.collect(Collectors.toList());
}
- private Schema icebergSchemaToPaimonSchema(IcebergSchema icebergSchema) {
+ private Schema icebergSchemaToPaimonSchema(
+ IcebergSchema icebergSchema, Map<String, String> options) {
// get iceberg current partition spec
int currentPartitionSpecId = icebergMetadata.defaultSpecId();
@@ -321,8 +326,7 @@ public class IcebergMigrator implements Migrator {
.map(IcebergPartitionField::name)
.collect(Collectors.toList());
- return new Schema(
- dataFields, partitionKeys, Collections.emptyList(),
Collections.emptyMap(), null);
+ return new Schema(dataFields, partitionKeys, Collections.emptyList(),
options, null);
}
private void checkAndFilterManifestFiles(
diff --git
a/paimon-core/src/test/java/org/apache/paimon/iceberg/migrate/IcebergMigrateTest.java
b/paimon-core/src/test/java/org/apache/paimon/iceberg/migrate/IcebergMigrateTest.java
index c8e61a7f6d..b888c2f0f4 100644
---
a/paimon-core/src/test/java/org/apache/paimon/iceberg/migrate/IcebergMigrateTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/iceberg/migrate/IcebergMigrateTest.java
@@ -801,6 +801,47 @@ public class IcebergMigrateTest {
record.toString().replaceAll("\\bRecord\\b",
"")));
}
+ @Test
+ public void testMigrateTableWithProperties() throws Exception {
+ // Create Iceberg table with custom properties
+ org.apache.iceberg.catalog.Catalog icebergCatalog =
createIcebergCatalog();
+ TableIdentifier icebergIdentifier = TableIdentifier.of(iceDatabase,
iceTable);
+
+ Map<String, String> tableProperties = new HashMap<>();
+ tableProperties.put("write.format.default", "parquet");
+ tableProperties.put("comment", "description");
+
+ Table icebergTable =
+ icebergCatalog
+ .buildTable(icebergIdentifier, iceSchema)
+ .withPartitionSpec(PartitionSpec.unpartitioned())
+ .withProperties(tableProperties)
+ .create();
+
+ // Verify Iceberg table has the properties set
+
assertThat(icebergTable.properties()).containsAllEntriesOf(tableProperties);
+ String format = "parquet";
+ writeInitialData(icebergTable, format, false);
+
+ // Execute migration
+ IcebergMigrator icebergMigrator =
+ new IcebergMigrator(
+ paiCatalog,
+ paiDatabase,
+ paiTable,
+ iceDatabase,
+ iceTable,
+ new Options(icebergProperties),
+ 1,
+ Collections.emptyMap());
+ icebergMigrator.executeMigrate();
+
+ // Verify Paimon table with the same properties
+ FileStoreTable paimonTable =
+ (FileStoreTable)
paiCatalog.getTable(Identifier.create(paiDatabase, paiTable));
+
assertThat(paimonTable.options()).containsAllEntriesOf(tableProperties);
+ }
+
private org.apache.iceberg.catalog.Catalog createIcebergCatalog() {
Map<String, String> icebergCatalogOptions = new HashMap<>();
icebergCatalogOptions.put("type", "hadoop");
