This is an automated email from the ASF dual-hosted git repository. vinish pushed a commit to branch 590-CatalogSync in repository https://gitbox.apache.org/repos/asf/incubator-xtable.git
commit f0c1469ea42761df551535516e9d69bfa880eb93 Author: Vinish Reddy <[email protected]> AuthorDate: Thu Dec 19 14:12:34 2024 -0800 Rebase with latest API branch --- .../xtable/conversion/ExternalCatalogConfig.java | 22 +++++++++++++++++----- .../xtable/conversion/TargetCatalogConfig.java | 2 +- .../catalog/ExternalCatalogConfigFactory.java | 4 ++-- .../xtable/iceberg/IcebergCatalogConfig.java | 2 +- .../catalog/TestCatalogConversionFactory.java | 4 ++-- .../conversion/TestConversionController.java | 2 +- .../org/apache/xtable/testutil/ITTestUtils.java | 2 +- .../apache/xtable/utilities/RunCatalogSync.java | 19 +++++++++++-------- .../src/test/resources/catalogConfig.yaml | 17 +++++++++-------- 9 files changed, 45 insertions(+), 29 deletions(-) diff --git a/xtable-api/src/main/java/org/apache/xtable/conversion/ExternalCatalogConfig.java b/xtable-api/src/main/java/org/apache/xtable/conversion/ExternalCatalogConfig.java index 16785ec6..148cb2fb 100644 --- a/xtable-api/src/main/java/org/apache/xtable/conversion/ExternalCatalogConfig.java +++ b/xtable-api/src/main/java/org/apache/xtable/conversion/ExternalCatalogConfig.java @@ -25,15 +25,27 @@ import lombok.Builder; import lombok.NonNull; import lombok.Value; -/** Defines the configuration for an external catalog. */ +/** + * Defines the configuration for an external catalog, user needs to populate at-least one of + * catalogType or catalogImpl + */ @Value @Builder -public class ExternalCatalogConfig implements CatalogConfig { +public class ExternalCatalogConfig { /** The name of the catalog, it also acts as a unique identifier for each catalog */ - @NonNull String catalogName; + @NonNull String catalogId; + + /** + * The type of the catalog. If the catalogType implementation exists in XTable, the implementation + * class will be inferred. + */ + String catalogType; - /** The implementation class path for the catalog */ - @NonNull String catalogImpl; + /** + * (Optional) A fully qualified class name that implements the interfaces for CatalogSyncClient, + * it can be used if the implementation for catalogType doesn't exist in XTable. + */ + String catalogImpl; /** * The properties for each catalog, used for providing any custom behaviour during catalog sync diff --git a/xtable-api/src/main/java/org/apache/xtable/conversion/TargetCatalogConfig.java b/xtable-api/src/main/java/org/apache/xtable/conversion/TargetCatalogConfig.java index ca6cec2d..46bd4600 100644 --- a/xtable-api/src/main/java/org/apache/xtable/conversion/TargetCatalogConfig.java +++ b/xtable-api/src/main/java/org/apache/xtable/conversion/TargetCatalogConfig.java @@ -32,7 +32,7 @@ import org.apache.xtable.model.catalog.CatalogTableIdentifier; @Builder public class TargetCatalogConfig { /** - * The tableIdentifiers(databaseName, tableName) that will be used when syncing {@link + * The tableIdentifiers(catalogName.databaseName.tableName) that will be used when syncing {@link * TargetTable} to the catalog. */ @NonNull CatalogTableIdentifier catalogTableIdentifier; diff --git a/xtable-core/src/main/java/org/apache/xtable/catalog/ExternalCatalogConfigFactory.java b/xtable-core/src/main/java/org/apache/xtable/catalog/ExternalCatalogConfigFactory.java index 09bf1566..f053df32 100644 --- a/xtable-core/src/main/java/org/apache/xtable/catalog/ExternalCatalogConfigFactory.java +++ b/xtable-core/src/main/java/org/apache/xtable/catalog/ExternalCatalogConfigFactory.java @@ -26,12 +26,12 @@ import org.apache.xtable.conversion.ExternalCatalogConfig; public class ExternalCatalogConfigFactory { public static ExternalCatalogConfig fromCatalogType( - String catalogType, String catalogName, Map<String, String> properties) { + String catalogType, String catalogId, Map<String, String> properties) { // TODO: Choose existing implementation based on catalogType. String catalogImpl = ""; return ExternalCatalogConfig.builder() .catalogImpl(catalogImpl) - .catalogName(catalogName) + .catalogId(catalogId) .catalogOptions(properties) .build(); } diff --git a/xtable-core/src/main/java/org/apache/xtable/iceberg/IcebergCatalogConfig.java b/xtable-core/src/main/java/org/apache/xtable/iceberg/IcebergCatalogConfig.java index b678bf00..e0ec7762 100644 --- a/xtable-core/src/main/java/org/apache/xtable/iceberg/IcebergCatalogConfig.java +++ b/xtable-core/src/main/java/org/apache/xtable/iceberg/IcebergCatalogConfig.java @@ -30,7 +30,7 @@ import org.apache.xtable.conversion.CatalogConfig; /** * Iceberg requires a catalog to perform any operation, if no catalog is provided the default * catalog (HadoopCatalog or storage based catalog) is used. For syncing iceberg to multiple - * catalogs, you can use {@link org.apache.xtable.catalog.ExternalCatalogConfig} instead which + * catalogs, you can use {@link org.apache.xtable.conversion.ExternalCatalogConfig} instead which * allows syncing the latest version of iceberg metadata to multiple catalogs. */ @Value diff --git a/xtable-core/src/test/java/org/apache/xtable/catalog/TestCatalogConversionFactory.java b/xtable-core/src/test/java/org/apache/xtable/catalog/TestCatalogConversionFactory.java index 6a6a5592..386ecd64 100644 --- a/xtable-core/src/test/java/org/apache/xtable/catalog/TestCatalogConversionFactory.java +++ b/xtable-core/src/test/java/org/apache/xtable/catalog/TestCatalogConversionFactory.java @@ -38,7 +38,7 @@ class TestCatalogConversionFactory { void createSourceForConfig() { ExternalCatalogConfig sourceCatalog = ExternalCatalogConfig.builder() - .catalogName("catalogName") + .catalogId("catalogId") .catalogImpl(TestCatalogImpl.class.getName()) .catalogOptions(Collections.emptyMap()) .build(); @@ -53,7 +53,7 @@ class TestCatalogConversionFactory { TargetCatalogConfig.builder() .catalogConfig( ExternalCatalogConfig.builder() - .catalogName("catalogName") + .catalogId("catalogId") .catalogImpl(TestCatalogImpl.class.getName()) .catalogOptions(Collections.emptyMap()) .build()) diff --git a/xtable-core/src/test/java/org/apache/xtable/conversion/TestConversionController.java b/xtable-core/src/test/java/org/apache/xtable/conversion/TestConversionController.java index 11dee760..c1fb9863 100644 --- a/xtable-core/src/test/java/org/apache/xtable/conversion/TestConversionController.java +++ b/xtable-core/src/test/java/org/apache/xtable/conversion/TestConversionController.java @@ -589,7 +589,7 @@ public class TestConversionController { return TargetCatalogConfig.builder() .catalogConfig( ExternalCatalogConfig.builder() - .catalogName("catalogName-" + suffix) + .catalogId("catalogId-" + suffix) .catalogImpl("catalogImpl-" + suffix) .catalogOptions(Collections.emptyMap()) .build()) diff --git a/xtable-core/src/test/java/org/apache/xtable/testutil/ITTestUtils.java b/xtable-core/src/test/java/org/apache/xtable/testutil/ITTestUtils.java index 4a113272..9a4f18ad 100644 --- a/xtable-core/src/test/java/org/apache/xtable/testutil/ITTestUtils.java +++ b/xtable-core/src/test/java/org/apache/xtable/testutil/ITTestUtils.java @@ -65,7 +65,7 @@ public class ITTestUtils { } @Override - public String getCatalogName() { + public String getCatalogId() { return null; } diff --git a/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunCatalogSync.java b/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunCatalogSync.java index 28e26dd9..b77ebd09 100644 --- a/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunCatalogSync.java +++ b/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunCatalogSync.java @@ -134,7 +134,7 @@ public class RunCatalogSync { Map<String, DatasetConfig.Catalog> catalogsByName = datasetConfig.getTargetCatalogs().stream() - .collect(Collectors.toMap(DatasetConfig.Catalog::getCatalogName, Function.identity())); + .collect(Collectors.toMap(DatasetConfig.Catalog::getCatalogId, Function.identity())); ExternalCatalogConfig sourceCatalogConfig = getCatalogConfig(datasetConfig.getSourceCatalog()); CatalogConversionSource catalogConversionSource = CatalogConversionFactory.createCatalogConversionSource(sourceCatalogConfig, hadoopConf); @@ -183,7 +183,7 @@ public class RunCatalogSync { targetCatalogTableIdentifier.getCatalogTableIdentifier()) .catalogConfig( getCatalogConfig( - catalogsByName.get(targetCatalogTableIdentifier.getCatalogName()))) + catalogsByName.get(targetCatalogTableIdentifier.getCatalogId()))) .build()); } ConversionConfig conversionConfig = @@ -211,10 +211,10 @@ public class RunCatalogSync { static ExternalCatalogConfig getCatalogConfig(DatasetConfig.Catalog catalog) { if (!StringUtils.isEmpty(catalog.getCatalogType())) { return ExternalCatalogConfigFactory.fromCatalogType( - catalog.getCatalogType(), catalog.getCatalogName(), catalog.getCatalogProperties()); + catalog.getCatalogType(), catalog.getCatalogId(), catalog.getCatalogProperties()); } else { return ExternalCatalogConfig.builder() - .catalogName(catalog.getCatalogName()) + .catalogId(catalog.getCatalogId()) .catalogImpl(catalog.getCatalogImpl()) .catalogOptions(catalog.getCatalogProperties()) .build(); @@ -264,8 +264,8 @@ public class RunCatalogSync { /** Configuration for catalog. */ @Data public static class Catalog { - /** A unique name for the catalog. */ - private String catalogName; + /** A user defined unique identifier for the catalog. */ + private String catalogId; /** * The type of the source catalog. This might be a specific type understood by XTable, such as * Hive, Glue etc. @@ -305,8 +305,11 @@ public class RunCatalogSync { @Data public static class TargetTableIdentifier { - /** name of the target catalog where the table will be created or updated */ - String catalogName; + /** + * The user defined unique identifier of the target {@link Catalog} where the table will be + * created or updated + */ + String catalogId; /** * The target table format (e.g., DELTA, HUDI, ICEBERG), specifying how the data will be * stored at the target. diff --git a/xtable-utilities/src/test/resources/catalogConfig.yaml b/xtable-utilities/src/test/resources/catalogConfig.yaml index f3f1757c..185f47b3 100644 --- a/xtable-utilities/src/test/resources/catalogConfig.yaml +++ b/xtable-utilities/src/test/resources/catalogConfig.yaml @@ -15,26 +15,26 @@ # limitations under the License. # sourceCatalog: - catalogName: "source-1" + catalogId: "source-1" catalogImpl: "org.apache.xtable.testutil.ITTestUtils$TestCatalogImpl" catalogProperties: key01: "value1" key02: "value2" key03: "value3" targetCatalogs: - - catalogName: "target-1" + - catalogId: "target-1" catalogImpl: "org.apache.xtable.testutil.ITTestUtils$TestCatalogImpl" catalogProperties: key11: "value1" key12: "value2" key13: "value3" - - catalogName: "target-2" + - catalogId: "target-2" catalogImpl: "org.apache.xtable.testutil.ITTestUtils$TestCatalogImpl" catalogProperties: key21: "value1" key22: "value2" key23: "value3" - - catalogName: "target-3" + - catalogId: "target-3" catalogImpl: "org.apache.xtable.testutil.ITTestUtils$TestCatalogImpl" catalogProperties: key31: "value1" @@ -46,12 +46,12 @@ datasets: databaseName: "source-database-1" tableName: "source-1" targetCatalogTableIdentifiers: - - catalogName: "target-1" + - catalogId: "target-1" tableFormat: "DELTA" catalogTableIdentifier: databaseName: "target-database-1" tableName: "target-tableName-1" - - catalogName: "target-2" + - catalogId: "target-2" tableFormat: "HUDI" catalogTableIdentifier: databaseName: "target-database-2" @@ -63,13 +63,14 @@ datasets: partitionSpec: cs_sold_date_sk:VALUE tableFormat: "HUDI" targetCatalogTableIdentifiers: - - catalogName: "target-2" + - catalogId: "target-2" tableFormat: "ICEBERG" catalogTableIdentifier: databaseName: "target-database-2" tableName: "target-tableName-2" - - catalogName: "target-3" + - catalogId: "target-3" tableFormat: "HUDI" catalogTableIdentifier: + catalogName: "default-catalog-2" databaseName: "target-database-3" tableName: "target-tableName-3"
