>From Hussain Towaileb <[email protected]>: Hussain Towaileb has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21265?usp=email )
Change subject: [NO ISSUE][EXT]: validate catalog endpoint on catalog creation ...................................................................... [NO ISSUE][EXT]: validate catalog endpoint on catalog creation Ext-ref: MB-71972 Change-Id: I328ee7d531959a0478d6c6a847bd60f13467b39e --- A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/negative/bad-catalog-endpoint/test.000.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/negative/bad-catalog-endpoint/test.010.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/negative/bad-catalog-endpoint/test.020.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/negative/bad-catalog-endpoint/test.030.ddl.sqlpp M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_iceberg.xml M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/iceberg/glue/GlueUtils.java M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/iceberg/biglake_metastore/BiglakeMetastoreUtils.java M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/IcebergUtils.java M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/nessie/NessieUtils.java M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/rest/RestUtils.java 10 files changed, 153 insertions(+), 0 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/65/21265/1 diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/negative/bad-catalog-endpoint/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/negative/bad-catalog-endpoint/test.000.ddl.sqlpp new file mode 100644 index 0000000..c06e326 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/negative/bad-catalog-endpoint/test.000.ddl.sqlpp @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +CREATE CATALOG myGlueCatalog +TYPE Iceberg +SOURCE NESSIE +WITH { + "warehouse": "s3://cbas-iceberg-playground/my-glue-catalog/warehouse/", + "namespace": "pharmacy_namespace" +}; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/negative/bad-catalog-endpoint/test.010.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/negative/bad-catalog-endpoint/test.010.ddl.sqlpp new file mode 100644 index 0000000..4a87226 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/negative/bad-catalog-endpoint/test.010.ddl.sqlpp @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +CREATE CATALOG myGlueCatalog +TYPE Iceberg +SOURCE NESSIE +WITH { + "warehouse": "s3://cbas-iceberg-playground/my-glue-catalog/warehouse/", + "namespace": "pharmacy_namespace", + "uri": "http://"; +}; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/negative/bad-catalog-endpoint/test.020.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/negative/bad-catalog-endpoint/test.020.ddl.sqlpp new file mode 100644 index 0000000..6b78a95 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/negative/bad-catalog-endpoint/test.020.ddl.sqlpp @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +CREATE CATALOG myGlueCatalog +TYPE Iceberg +SOURCE NESSIE +WITH { + "warehouse": "s3://cbas-iceberg-playground/my-glue-catalog/warehouse/", + "namespace": "pharmacy_namespace", + "uri": "hello" +}; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/negative/bad-catalog-endpoint/test.030.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/negative/bad-catalog-endpoint/test.030.ddl.sqlpp new file mode 100644 index 0000000..d92d555 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/negative/bad-catalog-endpoint/test.030.ddl.sqlpp @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +CREATE CATALOG myGlueCatalog +TYPE Iceberg +SOURCE NESSIE +WITH { + "warehouse": "s3://cbas-iceberg-playground/my-glue-catalog/warehouse/", + "namespace": "pharmacy_namespace", + "uri": "127.0.0.1" +}; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_iceberg.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_iceberg.xml index 553a1d1..9797ccb 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_iceberg.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_iceberg.xml @@ -44,6 +44,15 @@ </test-group> <test-group name="negative"> <test-case FilePath="iceberg/negative"> + <compilation-unit name="bad-catalog-endpoint"> + <output-dir compare="Text">bad-catalog-endpoint</output-dir> + <expected-error>ASX0049: Parameter(s) uri must be specified</expected-error> + <expected-error>ASX1108: External source error. java.net.URISyntaxException: Expected authority at index 7: http://</expected-error> + <expected-error>ASX1108: External source error. missing URI scheme. received: hello</expected-error> + <expected-error>ASX1108: External source error. missing URI scheme. received: 127.0.0.1</expected-error> + </compilation-unit> + </test-case> + <test-case FilePath="iceberg/negative"> <compilation-unit name="bad-catalog-type"> <output-dir compare="Text">bad-catalog-type</output-dir> <expected-error>ASX1228: Unsupported catalog type: 'FakeIcebergCatalog'</expected-error> diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/iceberg/glue/GlueUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/iceberg/glue/GlueUtils.java index edc9916..2e047d0 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/iceberg/glue/GlueUtils.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/iceberg/glue/GlueUtils.java @@ -30,6 +30,7 @@ import static org.apache.asterix.external.util.iceberg.IcebergConstants.Aws.REST_SIG4_SIGNING_NAME_PROPERTY_NAME; import static org.apache.asterix.external.util.iceberg.IcebergConstants.Aws.REST_SIG4_SIGNING_REGION; import static org.apache.asterix.external.util.iceberg.IcebergConstants.Aws.REST_SIG4_SIGNING_REGION_PROPERTY_NAME; +import static org.apache.asterix.external.util.iceberg.IcebergUtils.validateIcebergCatalogUri; import static org.apache.asterix.external.util.iceberg.IcebergUtils.validatePropertyExists; import static org.apache.iceberg.CatalogProperties.CATALOG_IMPL; import static org.apache.iceberg.CatalogProperties.URI; @@ -114,6 +115,7 @@ public static void validateGlueRestRequiredProperties(Map<String, String> catalogProperties) throws CompilationException { validatePropertyExists(catalogProperties, ICEBERG_URI_PROPERTY_KEY, PARAMETERS_REQUIRED); + validateIcebergCatalogUri(catalogProperties); validatePropertyExists(catalogProperties, REST_SIG4_SIGNING_REGION_PROPERTY_NAME, PARAMETERS_REQUIRED); } @@ -121,6 +123,7 @@ throws CompilationException { validatePropertyExists(catalogProperties, ICEBERG_WAREHOUSE_PROPERTY_KEY, PARAMETERS_REQUIRED); validatePropertyExists(catalogProperties, ICEBERG_URI_PROPERTY_KEY, PARAMETERS_REQUIRED); + validateIcebergCatalogUri(catalogProperties); validatePropertyExists(catalogProperties, REST_SIG4_SIGNING_NAME_PROPERTY_NAME, PARAMETERS_REQUIRED); validatePropertyExists(catalogProperties, REST_SIG4_SIGNING_REGION_PROPERTY_NAME, PARAMETERS_REQUIRED); } diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/iceberg/biglake_metastore/BiglakeMetastoreUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/iceberg/biglake_metastore/BiglakeMetastoreUtils.java index 1b28a29..acdbf60 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/iceberg/biglake_metastore/BiglakeMetastoreUtils.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/iceberg/biglake_metastore/BiglakeMetastoreUtils.java @@ -23,6 +23,7 @@ import static org.apache.asterix.external.util.iceberg.IcebergConstants.ICEBERG_WAREHOUSE_PROPERTY_KEY; import static org.apache.asterix.external.util.iceberg.IcebergConstants.Gcp.QUOTA_PROJECT_ID_HEADER_KEY; import static org.apache.asterix.external.util.iceberg.IcebergConstants.Gcp.QUOTA_PROJECT_ID_KEY; +import static org.apache.asterix.external.util.iceberg.IcebergUtils.validateIcebergCatalogUri; import static org.apache.asterix.external.util.iceberg.IcebergUtils.validatePropertyExists; import static org.apache.iceberg.CatalogProperties.CATALOG_IMPL; import static org.apache.iceberg.CatalogProperties.URI; @@ -49,6 +50,7 @@ public static void validateRequiredProperties(Map<String, String> catalogProperties) throws CompilationException { validatePropertyExists(catalogProperties, ICEBERG_URI_PROPERTY_KEY, PARAMETERS_REQUIRED); + validateIcebergCatalogUri(catalogProperties); validatePropertyExists(catalogProperties, ICEBERG_WAREHOUSE_PROPERTY_KEY, PARAMETERS_REQUIRED); validatePropertyExists(catalogProperties, QUOTA_PROJECT_ID_KEY, PARAMETERS_REQUIRED); } diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/IcebergUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/IcebergUtils.java index 6b16404..4006a65 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/IcebergUtils.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/IcebergUtils.java @@ -18,6 +18,7 @@ */ package org.apache.asterix.external.util.iceberg; +import static org.apache.asterix.common.exceptions.ErrorCode.EXTERNAL_SOURCE_ERROR; import static org.apache.asterix.common.exceptions.ErrorCode.UNSUPPORTED_ICEBERG_DATA_FORMAT; import static org.apache.asterix.external.util.aws.EnsureCloseClientsFactoryRegistry.FACTORY_INSTANCE_ID_KEY; import static org.apache.asterix.external.util.iceberg.IcebergConstants.ICEBERG_AVRO_FORMAT; @@ -28,6 +29,8 @@ import static org.apache.asterix.external.util.iceberg.IcebergConstants.ICEBERG_WAREHOUSE_PROPERTY_KEY; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -373,6 +376,30 @@ } } + public static void validateIcebergCatalogUri(Map<String, String> properties) throws CompilationException { + String value = properties.get(IcebergConstants.ICEBERG_URI_PROPERTY_KEY); + if (value == null) { + return; + } + if (value.isBlank()) { + throw CompilationException.create(EXTERNAL_SOURCE_ERROR, "received blank URI value"); + } + + try { + URI uri = new URI(value); + if (!uri.isAbsolute()) { + throw CompilationException.create(EXTERNAL_SOURCE_ERROR, "missing URI scheme. received: " + value); + } + + if (uri.getHost() == null || uri.getHost().isBlank()) { + throw CompilationException.create(EXTERNAL_SOURCE_ERROR, + "URI host cannot be blank. received: " + value); + } + } catch (URISyntaxException ex) { + throw CompilationException.create(EXTERNAL_SOURCE_ERROR, ex, ex.toString()); + } + } + public static void setWarehouseIfPresent(Map<String, String> catalogProperties) { String warehouse = catalogProperties.get(ICEBERG_WAREHOUSE_PROPERTY_KEY); if (warehouse != null && !warehouse.isEmpty()) { diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/nessie/NessieUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/nessie/NessieUtils.java index 0e36ac6..f0df588 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/nessie/NessieUtils.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/nessie/NessieUtils.java @@ -30,6 +30,7 @@ import static org.apache.asterix.external.util.http.HttpConstants.USERNAME_FIELD_NAME; import static org.apache.asterix.external.util.iceberg.IcebergConstants.ICEBERG_URI_PROPERTY_KEY; import static org.apache.asterix.external.util.iceberg.IcebergConstants.ICEBERG_WAREHOUSE_PROPERTY_KEY; +import static org.apache.asterix.external.util.iceberg.IcebergUtils.validateIcebergCatalogUri; import static org.apache.asterix.external.util.iceberg.IcebergUtils.validatePropertyExists; import static org.apache.asterix.external.util.iceberg.nessie.NessieConstants.NESSIE_AUTHENTICATION_BEARER_TOKEN_FIELD_NAME; import static org.apache.asterix.external.util.iceberg.nessie.NessieConstants.NESSIE_AUTHENTICATION_OAUTH2_ALLOWED_SCOPES; @@ -72,12 +73,14 @@ public static void validateRequiredProperties(Map<String, String> catalogProperties) throws CompilationException { validatePropertyExists(catalogProperties, ICEBERG_URI_PROPERTY_KEY, PARAMETERS_REQUIRED); + validateIcebergCatalogUri(catalogProperties); validatePropertyExists(catalogProperties, ICEBERG_WAREHOUSE_PROPERTY_KEY, PARAMETERS_REQUIRED); } public static void validateNessieRestRequiredProperties(Map<String, String> catalogProperties) throws CompilationException { validatePropertyExists(catalogProperties, ICEBERG_URI_PROPERTY_KEY, PARAMETERS_REQUIRED); + validateIcebergCatalogUri(catalogProperties); } public static void setAuthentication(Map<String, String> catalogProperties) throws CompilationException { diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/rest/RestUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/rest/RestUtils.java index 569955a..57121b8 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/rest/RestUtils.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/rest/RestUtils.java @@ -29,6 +29,7 @@ import static org.apache.asterix.external.util.http.HttpConstants.PASSWORD_FIELD_NAME; import static org.apache.asterix.external.util.http.HttpConstants.USERNAME_FIELD_NAME; import static org.apache.asterix.external.util.iceberg.IcebergConstants.ICEBERG_URI_PROPERTY_KEY; +import static org.apache.asterix.external.util.iceberg.IcebergUtils.validateIcebergCatalogUri; import static org.apache.asterix.external.util.iceberg.IcebergUtils.validatePropertyExists; import java.util.Map; @@ -60,6 +61,7 @@ public static void validateRequiredProperties(Map<String, String> catalogProperties) throws CompilationException { validatePropertyExists(catalogProperties, ICEBERG_URI_PROPERTY_KEY, PARAMETERS_REQUIRED); + validateIcebergCatalogUri(catalogProperties); } private static void setAuthentication(Map<String, String> catalogProperties) throws CompilationException { -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21265?usp=email To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings?usp=email Gerrit-MessageType: newchange Gerrit-Project: asterixdb Gerrit-Branch: lumina Gerrit-Change-Id: I328ee7d531959a0478d6c6a847bd60f13467b39e Gerrit-Change-Number: 21265 Gerrit-PatchSet: 1 Gerrit-Owner: Hussain Towaileb <[email protected]>
