>From Hussain Towaileb <[email protected]>: Hussain Towaileb has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20949?usp=email )
Change subject: [ASTERIXDB-3634][EXT]: Refactor iceberg code ...................................................................... [ASTERIXDB-3634][EXT]: Refactor iceberg code Ext-ref: MB-70659 Change-Id: I99e48e51ac2081fc899d89bcbc22f3f00ee072e6 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20949 Reviewed-by: Hussain Towaileb <[email protected]> Reviewed-by: Michael Blow <[email protected]> Integration-Tests: Jenkins <[email protected]> Tested-by: Hussain Towaileb <[email protected]> --- M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java M asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/IcebergUtils.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/IcebergCatalog.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/CatalogTupleTranslator.java 7 files changed, 62 insertions(+), 8 deletions(-) Approvals: Hussain Towaileb: Looks good to me, but someone else must approve; Verified Michael Blow: Looks good to me, approved Jenkins: Verified diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java index dd8d358..45d7bb2 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java @@ -348,6 +348,7 @@ NO_VALID_CREDENTIALS_PROVIDED_FOR_BIGLAKE_METASTORE_CATALOG(1240), INVALID_FRAME_BASED_MEMORY_BUDGET(1241), COLLECTION_IS_NOT_AN_ICEBERG_TABLE_COLLECTION(1242), + NOT_ICEBERG_CATALOG(1243), // Feed errors DATAFLOW_ILLEGAL_STATE(3001), diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties index 23a66ff..578cd06 100644 --- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties +++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties @@ -350,6 +350,7 @@ 1240 = No valid credentials provided to access Biglake Metastore catalog. 1241 = Invalid `%1$s` "%2$s" for frame size=%3$s. value should be >= %4$s * frame size: `%1$s` "%5$s" in %6$s 1242 = Collection '%1$s' is not an Iceberg table external collection. +1243 = Catalog '%1$s' is not an Iceberg catalog. # Feed Errors 3001 = Illegal state. diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/IcebergUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/IcebergUtils.java index 20b5781..ec8001f 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/IcebergUtils.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/IcebergUtils.java @@ -30,6 +30,7 @@ import java.io.IOException; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -51,11 +52,13 @@ import org.apache.asterix.external.util.iceberg.rest.RestUtils; import org.apache.asterix.om.types.ARecordType; import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.Table; import org.apache.iceberg.aws.AwsProperties; import org.apache.iceberg.aws.glue.GlueCatalog; import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.SupportsNamespaces; +import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.nessie.NessieCatalog; import org.apache.iceberg.rest.RESTCatalog; import org.apache.logging.log4j.LogManager; @@ -396,4 +399,55 @@ addTo.putIfAbsent(ICEBERG_CATALOG_PROPERTY_PREFIX_INTERNAL + entry.getKey(), entry.getValue()); } } + + public static List<Namespace> listNamespaces(Catalog catalog, CatalogConfig.IcebergCatalogSource source) { + return switch (source) { + case AWS_GLUE -> { + GlueCatalog glueCatalog = (GlueCatalog) catalog; + yield glueCatalog.listNamespaces(); + } + case REST, AWS_GLUE_REST, BIGLAKE_METASTORE, S3_TABLES, NESSIE_REST -> { + RESTCatalog restCatalog = (RESTCatalog) catalog; + yield restCatalog.listNamespaces(); + } + case NESSIE -> { + NessieCatalog nessieCatalog = (NessieCatalog) catalog; + yield nessieCatalog.listNamespaces(); + } + }; + } + + public static List<TableIdentifier> listTables(Catalog catalog, Namespace namespace, CatalogConfig.IcebergCatalogSource source) { + return switch (source) { + case AWS_GLUE -> { + GlueCatalog glueCatalog = (GlueCatalog) catalog; + yield glueCatalog.listTables(namespace); + } + case REST, AWS_GLUE_REST, BIGLAKE_METASTORE, S3_TABLES, NESSIE_REST -> { + RESTCatalog restCatalog = (RESTCatalog) catalog; + yield restCatalog.listTables(namespace); + } + case NESSIE -> { + NessieCatalog nessieCatalog = (NessieCatalog) catalog; + yield nessieCatalog.listTables(namespace); + } + }; + } + + public static Table loadTable(Catalog catalog, TableIdentifier tableIdentifier, CatalogConfig.IcebergCatalogSource source) { + return switch (source) { + case AWS_GLUE -> { + GlueCatalog glueCatalog = (GlueCatalog) catalog; + yield glueCatalog.loadTable(tableIdentifier); + } + case REST, AWS_GLUE_REST, BIGLAKE_METASTORE, S3_TABLES, NESSIE_REST -> { + RESTCatalog restCatalog = (RESTCatalog) catalog; + yield restCatalog.loadTable(tableIdentifier); + } + case NESSIE -> { + NessieCatalog nessieCatalog = (NessieCatalog) catalog; + yield nessieCatalog.loadTable(tableIdentifier); + } + }; + } } diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java index 02ffc25..f0d584d 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java @@ -124,7 +124,7 @@ } protected abstract IDatasourceFunction createFunction(MetadataProvider metadataProvider, - AlgebricksAbsolutePartitionConstraint locations); + AlgebricksAbsolutePartitionConstraint locations) throws AlgebricksException; protected AlgebricksAbsolutePartitionConstraint getLocations(IClusterStateManager csm, MetadataProvider md) { String[] locations = md.getClusterLocations().getLocations(); diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java index 8963ed1..b97460a 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java @@ -1029,7 +1029,7 @@ String catalogName = configuration.get(IcebergConstants.ICEBERG_CATALOG_NAME); IcebergCatalog catalog = (IcebergCatalog) MetadataManager.INSTANCE.getCatalog(getMetadataTxnContext(), catalogName); - IcebergCatalogDetails details = (IcebergCatalogDetails) catalog.getCatalogDetails(); + IcebergCatalogDetails details = catalog.getCatalogDetails(); IcebergUtils.putCatalogProperties(configuration, details.getProperties()); } } diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/IcebergCatalog.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/IcebergCatalog.java index 798f987..0060a60 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/IcebergCatalog.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/IcebergCatalog.java @@ -18,22 +18,21 @@ */ package org.apache.asterix.metadata.entities; -import org.apache.asterix.metadata.ICatalogDetails; import org.apache.asterix.metadata.utils.Creator; public class IcebergCatalog extends Catalog { private static final long serialVersionUID = 1L; - private final ICatalogDetails catalogDetails; + private final IcebergCatalogDetails catalogDetails; - public IcebergCatalog(String catalogName, String catalogType, ICatalogDetails catalogDetails, int pendingOp, + public IcebergCatalog(String catalogName, String catalogType, IcebergCatalogDetails catalogDetails, int pendingOp, Creator creator) { super(catalogName, catalogType, pendingOp, creator); this.catalogDetails = catalogDetails; } - public ICatalogDetails getCatalogDetails() { + public IcebergCatalogDetails getCatalogDetails() { return catalogDetails; } } diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/CatalogTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/CatalogTupleTranslator.java index 05dd790..a3403d8 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/CatalogTupleTranslator.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/CatalogTupleTranslator.java @@ -23,7 +23,6 @@ import java.util.Map; import org.apache.asterix.external.util.iceberg.IcebergUtils; -import org.apache.asterix.metadata.ICatalogDetails; import org.apache.asterix.metadata.bootstrap.CatalogEntity; import org.apache.asterix.metadata.bootstrap.MetadataRecordTypes; import org.apache.asterix.metadata.entities.Catalog; @@ -76,7 +75,7 @@ IACursor cursor = ((AOrderedList) catalogDetailsRecord .getValueByPos(MetadataRecordTypes.CATALOG_DETAILS_ARECORD_PROPERTIES_FIELD_INDEX)).getCursor(); Map<String, String> properties = TupleTranslatorUtils.getPropertiesFromIaCursor(cursor); - ICatalogDetails catalogDetails = new IcebergCatalogDetails(adapter, properties); + IcebergCatalogDetails catalogDetails = new IcebergCatalogDetails(adapter, properties); return new IcebergCatalog(catalogName, catalogType, catalogDetails, pendingOp, creator); } return new Catalog(catalogName, catalogType, pendingOp, creator); -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20949?usp=email To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings?usp=email Gerrit-MessageType: merged Gerrit-Project: asterixdb Gerrit-Branch: lumina Gerrit-Change-Id: I99e48e51ac2081fc899d89bcbc22f3f00ee072e6 Gerrit-Change-Number: 20949 Gerrit-PatchSet: 5 Gerrit-Owner: Hussain Towaileb <[email protected]> Gerrit-Reviewer: Ali Alsuliman <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Hussain Towaileb <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Michael Blow <[email protected]> Gerrit-Reviewer: Murtadha Hubail <[email protected]> Gerrit-Reviewer: Peeyush Gupta <[email protected]>
