>From Hussain Towaileb <[email protected]>: Hussain Towaileb has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17806 )
Change subject: [ASTERIXDB-3269][EXT]: Handle root properly when computed field is at first segment ...................................................................... [ASTERIXDB-3269][EXT]: Handle root properly when computed field is at first segment Change-Id: Idc97e6eef1d13953f16a37b340f4ba13983ecd74 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17806 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Hussain Towaileb <[email protected]> Reviewed-by: Ian Maxon <[email protected]> --- M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetOnePartitionTest.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetOnePartitionTest.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml A asterixdb/asterix-app/data/json/external-filter/computed-field-at-start/bar-2023-01-01/data.json A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.010.query.sqlpp M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.000.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.000.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.010.query.sqlpp M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.999.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.999.ddl.sqlpp M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java A asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/computed-field-at-start/result.010.adm A asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/result.010.adm A asterixdb/asterix-app/data/json/external-filter/computed-field-at-start/foo-2023-01-01/data.json 19 files changed, 275 insertions(+), 7 deletions(-) Approvals: Ian Maxon: Looks good to me, approved Hussain Towaileb: Looks good to me, but someone else must approve Jenkins: Verified; Verified diff --git a/asterixdb/asterix-app/data/json/external-filter/computed-field-at-start/bar-2023-01-01/data.json b/asterixdb/asterix-app/data/json/external-filter/computed-field-at-start/bar-2023-01-01/data.json new file mode 100644 index 0000000..4b16428 --- /dev/null +++ b/asterixdb/asterix-app/data/json/external-filter/computed-field-at-start/bar-2023-01-01/data.json @@ -0,0 +1 @@ +{ "id": 2 } \ No newline at end of file diff --git a/asterixdb/asterix-app/data/json/external-filter/computed-field-at-start/foo-2023-01-01/data.json b/asterixdb/asterix-app/data/json/external-filter/computed-field-at-start/foo-2023-01-01/data.json new file mode 100644 index 0000000..7052c42 --- /dev/null +++ b/asterixdb/asterix-app/data/json/external-filter/computed-field-at-start/foo-2023-01-01/data.json @@ -0,0 +1 @@ +{ "id": 1 } \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java index 0db9a2a..90f46ad 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java @@ -19,6 +19,7 @@ package org.apache.asterix.test.external_dataset; import static org.apache.asterix.test.external_dataset.aws.AwsS3ExternalDatasetTest.BOM_FILE_CONTAINER; +import static org.apache.asterix.test.external_dataset.aws.AwsS3ExternalDatasetTest.DYNAMIC_PREFIX_AT_START_CONTAINER; import static org.apache.asterix.test.external_dataset.aws.AwsS3ExternalDatasetTest.FIXED_DATA_CONTAINER; import static org.apache.asterix.test.external_dataset.parquet.BinaryFileConverterUtil.BINARY_GEN_BASEDIR; @@ -69,6 +70,7 @@ public static final int OVER_1000_OBJECTS_COUNT = 2999; private static Uploader playgroundDataLoader; + private static Uploader dynamicPrefixAtStartDataLoader; private static Uploader fixedDataLoader; private static Uploader mixedDataLoader; private static Uploader bomFileLoader; @@ -118,9 +120,10 @@ TSV_DATA_PATH = tsvDataPath; } - public static void setUploaders(Uploader playgroundDataLoader, Uploader fixedDataLoader, Uploader mixedDataLoader, - Uploader bomFileLoader) { + public static void setUploaders(Uploader playgroundDataLoader, Uploader dynamicPrefixAtStartDataLoader, + Uploader fixedDataLoader, Uploader mixedDataLoader, Uploader bomFileLoader) { ExternalDatasetTestUtils.playgroundDataLoader = playgroundDataLoader; + ExternalDatasetTestUtils.dynamicPrefixAtStartDataLoader = dynamicPrefixAtStartDataLoader; ExternalDatasetTestUtils.fixedDataLoader = fixedDataLoader; ExternalDatasetTestUtils.mixedDataLoader = mixedDataLoader; ExternalDatasetTestUtils.bomFileLoader = bomFileLoader; @@ -158,6 +161,23 @@ } /** + * Special container where dynamic prefix is the first segment + */ + public static void prepareDynamicPrefixAtStartContainer() { + LOGGER.info("Loading dynamic prefix data to " + DYNAMIC_PREFIX_AT_START_CONTAINER); + + // Files data + String path = + Paths.get(JSON_DATA_PATH, "external-filter", "computed-field-at-start", "foo-2023-01-01", "data.json") + .toString(); + dynamicPrefixAtStartDataLoader.upload("foo-2023-01-01/data.json", path, true, false); + + path = Paths.get(JSON_DATA_PATH, "external-filter", "computed-field-at-start", "bar-2023-01-01", "data.json") + .toString(); + dynamicPrefixAtStartDataLoader.upload("bar-2023-01-01/data.json", path, true, false); + } + + /** * This bucket is being filled by fixed data, a test is counting all records in this bucket. If this bucket is * changed, the test case will fail and its result will need to be updated each time */ diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetOnePartitionTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetOnePartitionTest.java index c3f22a4..86d03a1 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetOnePartitionTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetOnePartitionTest.java @@ -44,6 +44,8 @@ ONLY_TESTS = "only_external_dataset.xml"; TEST_CONFIG_FILE_NAME = "src/test/resources/cc-single.conf"; PREPARE_BUCKET = AwsS3ExternalDatasetOnePartitionTest::prepareS3Bucket; + PREPARE_DYNAMIC_PREFIX_AT_START_BUCKET = + AwsS3ExternalDatasetOnePartitionTest::prepareDynamicPrefixAtStartContainer; PREPARE_FIXED_DATA_BUCKET = AwsS3ExternalDatasetOnePartitionTest::prepareFixedDataBucket; PREPARE_MIXED_DATA_BUCKET = AwsS3ExternalDatasetOnePartitionTest::prepareMixedDataBucket; PREPARE_BOM_FILE_BUCKET = AwsS3ExternalDatasetOnePartitionTest::prepareBomDataBucket; @@ -54,6 +56,9 @@ private static void prepareS3Bucket() { } + private static void prepareDynamicPrefixAtStartContainer() { + } + private static void prepareFixedDataBucket() { } diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java index 246ea13..532da56 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java @@ -115,6 +115,7 @@ static String ONLY_TESTS; static String TEST_CONFIG_FILE_NAME; static Runnable PREPARE_BUCKET; + static Runnable PREPARE_DYNAMIC_PREFIX_AT_START_BUCKET; static Runnable PREPARE_FIXED_DATA_BUCKET; static Runnable PREPARE_MIXED_DATA_BUCKET; static Runnable PREPARE_BOM_FILE_BUCKET; @@ -144,6 +145,7 @@ protected TestCaseContext tcCtx; public static final String PLAYGROUND_CONTAINER = "playground"; + public static final String DYNAMIC_PREFIX_AT_START_CONTAINER = "dynamic-prefix-at-start-container"; public static final String FIXED_DATA_CONTAINER = "fixed-data"; // Do not use, has fixed data public static final String INCLUDE_EXCLUDE_CONTAINER = "include-exclude"; public static final String BOM_FILE_CONTAINER = "bom-file-container"; @@ -151,6 +153,8 @@ public static final PutObjectRequest.Builder playgroundBuilder = PutObjectRequest.builder().bucket(PLAYGROUND_CONTAINER); + public static final PutObjectRequest.Builder dynamicPrefixAtStartBuilder = + PutObjectRequest.builder().bucket(DYNAMIC_PREFIX_AT_START_CONTAINER); public static final PutObjectRequest.Builder fixedDataBuilder = PutObjectRequest.builder().bucket(FIXED_DATA_CONTAINER); public static final PutObjectRequest.Builder includeExcludeBuilder = @@ -166,7 +170,6 @@ } // iceberg - private static final Schema SCHEMA = new Schema(required(1, "id", Types.IntegerType.get()), required(2, "data", Types.StringType.get())); private static final Configuration CONF = new Configuration(); @@ -348,6 +351,7 @@ ONLY_TESTS = "only_external_dataset.xml"; TEST_CONFIG_FILE_NAME = "src/main/resources/cc.conf"; PREPARE_BUCKET = ExternalDatasetTestUtils::preparePlaygroundContainer; + PREPARE_DYNAMIC_PREFIX_AT_START_BUCKET = ExternalDatasetTestUtils::prepareDynamicPrefixAtStartContainer; PREPARE_FIXED_DATA_BUCKET = ExternalDatasetTestUtils::prepareFixedDataContainer; PREPARE_MIXED_DATA_BUCKET = ExternalDatasetTestUtils::prepareMixedDataContainer; PREPARE_BOM_FILE_BUCKET = ExternalDatasetTestUtils::prepareBomFileContainer; @@ -397,6 +401,7 @@ .endpointOverride(endpoint); client = builder.build(); client.createBucket(CreateBucketRequest.builder().bucket(PLAYGROUND_CONTAINER).build()); + client.createBucket(CreateBucketRequest.builder().bucket(DYNAMIC_PREFIX_AT_START_CONTAINER).build()); client.createBucket(CreateBucketRequest.builder().bucket(FIXED_DATA_CONTAINER).build()); client.createBucket(CreateBucketRequest.builder().bucket(INCLUDE_EXCLUDE_CONTAINER).build()); client.createBucket(CreateBucketRequest.builder().bucket(BOM_FILE_CONTAINER).build()); @@ -405,9 +410,11 @@ // Create the bucket and upload some json files setDataPaths(JSON_DATA_PATH, CSV_DATA_PATH, TSV_DATA_PATH); - setUploaders(AwsS3ExternalDatasetTest::loadPlaygroundData, AwsS3ExternalDatasetTest::loadFixedData, + setUploaders(AwsS3ExternalDatasetTest::loadPlaygroundData, + AwsS3ExternalDatasetTest::loadDynamicPrefixAtStartData, AwsS3ExternalDatasetTest::loadFixedData, AwsS3ExternalDatasetTest::loadMixedData, AwsS3ExternalDatasetTest::loadBomData); PREPARE_BUCKET.run(); + PREPARE_DYNAMIC_PREFIX_AT_START_BUCKET.run(); PREPARE_FIXED_DATA_BUCKET.run(); PREPARE_MIXED_DATA_BUCKET.run(); PREPARE_BOM_FILE_BUCKET.run(); @@ -418,6 +425,10 @@ client.putObject(playgroundBuilder.key(key).build(), getRequestBody(content, fromFile, gzipped)); } + private static void loadDynamicPrefixAtStartData(String key, String content, boolean fromFile, boolean gzipped) { + client.putObject(dynamicPrefixAtStartBuilder.key(key).build(), getRequestBody(content, fromFile, gzipped)); + } + private static void loadFixedData(String key, String content, boolean fromFile, boolean gzipped) { client.putObject(fixedDataBuilder.key(key).build(), getRequestBody(content, fromFile, gzipped)); } diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetOnePartitionTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetOnePartitionTest.java index 59c375a..9f9e783 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetOnePartitionTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetOnePartitionTest.java @@ -41,6 +41,8 @@ ONLY_TESTS = "only_external_dataset.xml"; TEST_CONFIG_FILE_NAME = "src/test/resources/cc-single.conf"; PREPARE_PLAYGROUND_CONTAINER = AzureBlobStorageExternalDatasetOnePartitionTest::preparePlaygroundContainer; + PREPARE_DYNAMIC_PREFIX_AT_START_CONTAINER = + AzureBlobStorageExternalDatasetOnePartitionTest::prepareDynamicPrefixAtStartContainer; PREPARE_FIXED_DATA_CONTAINER = AzureBlobStorageExternalDatasetOnePartitionTest::prepareFixedDataContainer; PREPARE_INCLUDE_EXCLUDE_CONTAINER = AzureBlobStorageExternalDatasetOnePartitionTest::prepareMixedDataContainer; PREPARE_BOM_FILE_BUCKET = AzureBlobStorageExternalDatasetOnePartitionTest::prepareBomDataContainer; @@ -50,6 +52,9 @@ private static void preparePlaygroundContainer() { } + private static void prepareDynamicPrefixAtStartContainer() { + } + private static void prepareFixedDataContainer() { } diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java index 08f3816..9858e56 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java @@ -85,6 +85,7 @@ static String ONLY_TESTS; static String TEST_CONFIG_FILE_NAME; static Runnable PREPARE_PLAYGROUND_CONTAINER; + static Runnable PREPARE_DYNAMIC_PREFIX_AT_START_CONTAINER; static Runnable PREPARE_FIXED_DATA_CONTAINER; static Runnable PREPARE_INCLUDE_EXCLUDE_CONTAINER; static Runnable PREPARE_BOM_FILE_BUCKET; @@ -98,6 +99,7 @@ // Region, container and definitions private static final String PLAYGROUND_CONTAINER = "playground"; + private static final String DYNAMIC_PREFIX_AT_START_CONTAINER = "dynamic-prefix-at-start-container"; private static final String FIXED_DATA_CONTAINER = "fixed-data"; // Do not use, has fixed data private static final String INCLUDE_EXCLUDE_CONTAINER = "include-exclude"; private static final String BOM_FILE_CONTAINER = "bom-file-container"; @@ -108,6 +110,7 @@ // Create a BlobServiceClient object which will be used to create a container client private static BlobServiceClient blobServiceClient; private static BlobContainerClient playgroundContainer; + private static BlobContainerClient dynamicPrefixAtStartContainer; private static BlobContainerClient publicAccessContainer; private static BlobContainerClient fixedDataContainer; private static BlobContainerClient mixedDataContainer; @@ -140,6 +143,7 @@ ONLY_TESTS = "only_external_dataset.xml"; TEST_CONFIG_FILE_NAME = "src/main/resources/cc.conf"; PREPARE_PLAYGROUND_CONTAINER = ExternalDatasetTestUtils::preparePlaygroundContainer; + PREPARE_DYNAMIC_PREFIX_AT_START_CONTAINER = ExternalDatasetTestUtils::prepareDynamicPrefixAtStartContainer; PREPARE_FIXED_DATA_CONTAINER = ExternalDatasetTestUtils::prepareFixedDataContainer; PREPARE_INCLUDE_EXCLUDE_CONTAINER = ExternalDatasetTestUtils::prepareMixedDataContainer; PREPARE_BOM_FILE_BUCKET = ExternalDatasetTestUtils::prepareBomFileContainer; @@ -177,6 +181,7 @@ LOGGER.info("Creating containers"); playgroundContainer = blobServiceClient.createBlobContainer(PLAYGROUND_CONTAINER); + dynamicPrefixAtStartContainer = blobServiceClient.createBlobContainer(DYNAMIC_PREFIX_AT_START_CONTAINER); fixedDataContainer = blobServiceClient.createBlobContainer(FIXED_DATA_CONTAINER); mixedDataContainer = blobServiceClient.createBlobContainer(INCLUDE_EXCLUDE_CONTAINER); bomContainer = blobServiceClient.createBlobContainer(BOM_FILE_CONTAINER); @@ -195,9 +200,11 @@ // Create the bucket and upload some json files setDataPaths(JSON_DATA_PATH, CSV_DATA_PATH, TSV_DATA_PATH); setUploaders(AzureBlobStorageExternalDatasetTest::loadPlaygroundData, + AzureBlobStorageExternalDatasetTest::loadDynamicPrefixAtStartData, AzureBlobStorageExternalDatasetTest::loadFixedData, AzureBlobStorageExternalDatasetTest::loadMixedData, AzureBlobStorageExternalDatasetTest::loadBomData); PREPARE_PLAYGROUND_CONTAINER.run(); + PREPARE_DYNAMIC_PREFIX_AT_START_CONTAINER.run(); PREPARE_FIXED_DATA_CONTAINER.run(); PREPARE_INCLUDE_EXCLUDE_CONTAINER.run(); PREPARE_BOM_FILE_BUCKET.run(); @@ -240,6 +247,35 @@ } } + private static void loadDynamicPrefixAtStartData(String key, String content, boolean fromFile, boolean gzipped) { + if (!fromFile) { + try (ByteArrayInputStream inputStream = new ByteArrayInputStream(content.getBytes())) { + dynamicPrefixAtStartContainer.getBlobClient(key).upload(inputStream, inputStream.available()); + } catch (IOException ex) { + throw new IllegalArgumentException(ex.toString()); + } + } else { + if (!gzipped) { + dynamicPrefixAtStartContainer.getBlobClient(key).uploadFromFile(content); + } else { + try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + GZIPOutputStream gzipOutputStream = new GZIPOutputStream(byteArrayOutputStream)) { + gzipOutputStream.write(Files.readAllBytes(Paths.get(content))); + gzipOutputStream.close(); // Need to close or data will be invalid + byte[] gzipBytes = byteArrayOutputStream.toByteArray(); + + try (ByteArrayInputStream inputStream = new ByteArrayInputStream(gzipBytes)) { + dynamicPrefixAtStartContainer.getBlobClient(key).upload(inputStream, inputStream.available()); + } catch (IOException ex) { + throw new IllegalArgumentException(ex.toString()); + } + } catch (IOException ex) { + throw new IllegalArgumentException(ex.toString()); + } + } + } + } + private static void loadFixedData(String key, String content, boolean fromFile, boolean gzipped) { if (!fromFile) { try (ByteArrayInputStream inputStream = new ByteArrayInputStream(content.getBytes())) { @@ -417,6 +453,7 @@ private static void deleteContainersSilently() { deleteContainerSilently(PLAYGROUND_CONTAINER); + deleteContainerSilently(DYNAMIC_PREFIX_AT_START_CONTAINER); deleteContainerSilently(FIXED_DATA_CONTAINER); deleteContainerSilently(PUBLIC_ACCESS_CONTAINER); deleteContainerSilently(INCLUDE_EXCLUDE_CONTAINER); diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.000.ddl.sqlpp new file mode 100644 index 0000000..cb78e58 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.000.ddl.sqlpp @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +DROP DATAVERSE test IF EXISTS; +CREATE DATAVERSE test; +USE test; + +CREATE TYPE test AS { +}; + +CREATE EXTERNAL DATASET test1(test) USING %adapter% ( + %template%, + ("container"="dynamic-prefix-at-start-container"), + ("definition"="foo-{year:int}-{month:int}-{day:int}/"), + ("embed-filter-values" = "true"), + ("format"="json") +); + diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.010.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.010.query.sqlpp new file mode 100644 index 0000000..7d47884 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.010.query.sqlpp @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// param max-warnings:json=10 + +USE test; + +SELECT value t +FROM test1 t +order by t.id; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.999.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.999.ddl.sqlpp new file mode 100644 index 0000000..36b2bab --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.999.ddl.sqlpp @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +DROP DATAVERSE test IF EXISTS; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.000.ddl.sqlpp new file mode 100644 index 0000000..3f09568 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.000.ddl.sqlpp @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +DROP DATAVERSE test IF EXISTS; +CREATE DATAVERSE test; +USE test; + +CREATE TYPE test AS { +}; + +CREATE EXTERNAL DATASET test1(test) USING %adapter% ( + %template%, + ("container"="playground"), + ("definition"="parquet-data/foo-{year:int}-{month:int}-{day:int}/"), + ("embed-filter-values" = "true"), + ("format"="parquet") +); + diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.010.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.010.query.sqlpp new file mode 100644 index 0000000..7d47884 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.010.query.sqlpp @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// param max-warnings:json=10 + +USE test; + +SELECT value t +FROM test1 t +order by t.id; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.999.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.999.ddl.sqlpp new file mode 100644 index 0000000..36b2bab --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.999.ddl.sqlpp @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +DROP DATAVERSE test IF EXISTS; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/computed-field-at-start/result.010.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/computed-field-at-start/result.010.adm new file mode 100644 index 0000000..841ede8 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/computed-field-at-start/result.010.adm @@ -0,0 +1 @@ +{ "id": 1, "month": 1, "year": 2023, "day": 1 } \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/result.010.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/result.010.adm new file mode 100644 index 0000000..8c86668 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/result.010.adm @@ -0,0 +1 @@ +{ "id": 2, "month": 1, "year": 2023, "day": 1 } \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml index 55764ed..5809912 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml @@ -302,6 +302,12 @@ <output-dir compare="Text">computed-field-segment-pattern-mismatch</output-dir> </compilation-unit> </test-case> + <test-case FilePath="external-dataset/common/dynamic-prefixes"> + <compilation-unit name="computed-field-at-start"> + <placeholder name="adapter" value="AZUREBLOB" /> + <output-dir compare="Text">computed-field-at-start</output-dir> + </compilation-unit> + </test-case> <!-- <test-case FilePath="external-dataset/common/dynamic-prefixes/parquet"> <compilation-unit name="computed-field-segment-pattern-mismatch"> diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml index 623e7bc..54758e3 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml @@ -274,6 +274,12 @@ <output-dir compare="Text">computed-field-segment-pattern-mismatch</output-dir> </compilation-unit> </test-case> + <test-case FilePath="external-dataset/common/dynamic-prefixes"> + <compilation-unit name="computed-field-at-start"> + <placeholder name="adapter" value="S3" /> + <output-dir compare="Text">computed-field-at-start</output-dir> + </compilation-unit> + </test-case> <test-case FilePath="external-dataset/common/dynamic-prefixes/parquet"> <compilation-unit name="one-field"> <placeholder name="adapter" value="S3" /> diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java index 299d0e4..2edf326 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java @@ -210,7 +210,7 @@ // remove last "/" and append it only if needed root = builder.toString(); - root = root.substring(0, root.length() - 1); + root = root.isEmpty() ? root : root.substring(0, root.length() - 1); root = ExternalDataUtils.appendSlash(root, endsWithSlash); } diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java index 6902175..d282ab4 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java @@ -762,7 +762,7 @@ String definition = configuration.get(ExternalDataConstants.DEFINITION_FIELD_NAME); String subPath = configuration.get(ExternalDataConstants.SUBPATH); - boolean hasRoot = root != null && !root.isEmpty(); + boolean hasRoot = root != null; boolean hasDefinition = definition != null && !definition.isEmpty(); boolean hasSubPath = subPath != null && !subPath.isEmpty(); @@ -794,7 +794,7 @@ } public static String appendSlash(String string, boolean appendSlash) { - return appendSlash ? string + (!string.endsWith("/") ? "/" : "") : string; + return appendSlash && !string.isEmpty() ? string + (!string.endsWith("/") ? "/" : "") : string; } /** -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17806 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: Idc97e6eef1d13953f16a37b340f4ba13983ecd74 Gerrit-Change-Number: 17806 Gerrit-PatchSet: 4 Gerrit-Owner: Hussain Towaileb <[email protected]> Gerrit-Reviewer: Hussain Towaileb <[email protected]> Gerrit-Reviewer: Ian Maxon <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-MessageType: merged
