>From Ayush Tripathi <[email protected]>:
Attention is currently required from: Murtadha Hubail.
Hello Murtadha Hubail,
I'd like you to do a code review.
Please visit
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19008
to review the following change.
Change subject: [ASTERIXDB-3503][EXT] Deltalake invalid type error code change
......................................................................
[ASTERIXDB-3503][EXT] Deltalake invalid type error code change
- user model changes: no
- storage format changes: no
- interface changes: no
Change-Id: Ic0c12e32b85987cbc14d28e83fb2be644b922590
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18952
Integration-Tests: Jenkins <[email protected]>
Reviewed-by: Ayush Tripathi <[email protected]>
Reviewed-by: Murtadha Hubail <[email protected]>
Tested-by: Murtadha Hubail <[email protected]>
---
M
asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
M
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-multiple-file-read/deltalake-multiple_file_read.00.ddl.sqlpp
M
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-modified-data/deltalake-modified-data.00.ddl.sqlpp
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-invalid-file-format/deltalake-invalid-file-format.00.ddl.sqlpp
M asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
M
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-empty/deltalake-empty.00.ddl.sqlpp
8 files changed, 72 insertions(+), 10 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/08/19008/1
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-empty/deltalake-empty.00.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-empty/deltalake-empty.00.ddl.sqlpp
index f21f013..2e77beb 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-empty/deltalake-empty.00.ddl.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-empty/deltalake-empty.00.ddl.sqlpp
@@ -26,7 +26,7 @@
CREATE TYPE DeltalakeTableType as {
};
- CREATE EXTERNAL DATASET DeltalakeDataset(DeltalakeTableType) USING %adapter%
+ CREATE EXTERNAL COLLECTION DeltalakeDataset(DeltalakeTableType) USING
%adapter%
(
%template%,
("container"="playground"),
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-invalid-file-format/deltalake-invalid-file-format.00.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-invalid-file-format/deltalake-invalid-file-format.00.ddl.sqlpp
new file mode 100644
index 0000000..98799e1
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-invalid-file-format/deltalake-invalid-file-format.00.ddl.sqlpp
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
+ CREATE DATAVERSE test;
+
+ USE test;
+
+
+ CREATE TYPE DeltalakeTableType as {
+ };
+
+ CREATE EXTERNAL COLLECTION DeltalakeDataset(DeltalakeTableType) USING
%adapter%
+ (
+ %template%,
+ ("container"="playground"),
+ ("definition"="delta-data/my-table-empty"),
+ ("table-format" = "delta"),
+ ("format" = "avro")
+ );
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-modified-data/deltalake-modified-data.00.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-modified-data/deltalake-modified-data.00.ddl.sqlpp
index 8adab77..9238da6 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-modified-data/deltalake-modified-data.00.ddl.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-modified-data/deltalake-modified-data.00.ddl.sqlpp
@@ -26,7 +26,7 @@
CREATE TYPE DeltalakeTableType as {
};
- CREATE EXTERNAL DATASET DeltalakeDataset(DeltalakeTableType) USING %adapter%
+ CREATE EXTERNAL COLLECTION DeltalakeDataset(DeltalakeTableType) USING
%adapter%
(
%template%,
("container"="playground"),
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-multiple-file-read/deltalake-multiple_file_read.00.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-multiple-file-read/deltalake-multiple_file_read.00.ddl.sqlpp
index 61c5c3a..9aa0380 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-multiple-file-read/deltalake-multiple_file_read.00.ddl.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-multiple-file-read/deltalake-multiple_file_read.00.ddl.sqlpp
@@ -26,7 +26,7 @@
CREATE TYPE DeltalakeTableType as {
};
- CREATE EXTERNAL DATASET DeltalakeDataset(DeltalakeTableType) USING %adapter%
+ CREATE EXTERNAL COLLECTION DeltalakeDataset(DeltalakeTableType) USING
%adapter%
(
%template%,
("container"="playground"),
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index 4e94791..2c7af49 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@ -321,6 +321,13 @@
</compilation-unit>
</test-case>
<test-case FilePath="external-dataset">
+ <compilation-unit name="common/deltalake-invalid-file-format">
+ <placeholder name="adapter" value="S3" />
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>Supported file format for 'delta' tables is 'parquet',
but 'avro' was provided.</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
<compilation-unit name="common/avro/avro-types/avro-map">
<placeholder name="adapter" value="S3" />
<output-dir compare="Text">common/avro/avro-types/avro-map</output-dir>
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index 985a4f6..15c37ea 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.common.exceptions;
+import org.apache.asterix.common.utils.IdentifierUtil;
import org.apache.hyracks.api.exceptions.IError;
import org.apache.hyracks.api.util.ErrorMessageUtil;
@@ -97,8 +98,6 @@
UNSUPPORTED_COLUMN_TYPE(67),
INVALID_KEY_TYPE(68),
FAILED_TO_READ_KEY(69),
- INVALID_DELTA_PARAMETER(70),
-
UNSUPPORTED_JRE(100),
EXTERNAL_UDF_RESULT_TYPE_ERROR(200),
@@ -304,6 +303,7 @@
DUPLICATE_FIELD_IN_PRIMARY_KEY(1198),
INCOMPATIBLE_FIELDS_IN_PRIMARY_KEY(1199),
PREFIX_SHOULD_NOT_START_WITH_SLASH(1200),
+ INVALID_DELTA_TABLE_FORMAT(1201),
// Feed errors
DATAFLOW_ILLEGAL_STATE(3001),
@@ -461,8 +461,8 @@
}
private static class ErrorMessageMapHolder {
- private static final String[] enumMessages =
- ErrorMessageUtil.defineMessageEnumOrdinalMap(values(),
RESOURCE_PATH);
+ private static final String[] enumMessages = IdentifierUtil
+
.replaceIdentifiers(ErrorMessageUtil.defineMessageEnumOrdinalMap(values(),
RESOURCE_PATH));
private static String get(ErrorCode errorCode) {
return enumMessages[errorCode.ordinal()];
diff --git
a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index b5c06e3..8e3ccff 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -41,7 +41,6 @@
1009 = A returning expression cannot contain dataset access
37,1091 = Type mismatch: expected value of type %1$s, but got the value of
type %2$s
51 = Incomparable input types: %1$s and %2$s
-70 = Table Type 'delta' supports parquet file formats only.
# Data errors
6 = Invalid format for %1$s in %2$s
@@ -306,6 +305,7 @@
1198 = Duplicate field '%1$s' in primary key
1199 = Fields '%1$s' and '%2$s' are incompatible for primary key
1200 = Prefix should not start with "/". Prefix: '%1$s'
+1201 = Supported file format for 'delta' tables is 'parquet', but '%1$s' was
provided.
# Feed Errors
3001 = Illegal state.
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 8d33261..b4314b9 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -473,7 +473,7 @@
}
if (configuration.containsKey(ExternalDataConstants.TABLE_FORMAT)) {
- if
(configuration.get(ExternalDataConstants.TABLE_FORMAT).equals(ExternalDataConstants.FORMAT_DELTA))
{
+ if (isDeltaTable(configuration)) {
configuration.put(ExternalDataConstants.KEY_PARSER,
ExternalDataConstants.FORMAT_NOOP);
configuration.put(ExternalDataConstants.KEY_FORMAT,
ExternalDataConstants.FORMAT_PARQUET);
}
@@ -494,7 +494,8 @@
public static void validateDeltaTableProperties(Map<String, String>
configuration) throws CompilationException {
if (!(configuration.get(ExternalDataConstants.KEY_FORMAT) == null
||
configuration.get(ExternalDataConstants.KEY_FORMAT).equals(ExternalDataConstants.FORMAT_PARQUET)))
{
- throw new CompilationException(ErrorCode.INVALID_DELTA_PARAMETER);
+ throw new
CompilationException(ErrorCode.INVALID_DELTA_TABLE_FORMAT,
+ configuration.get(ExternalDataConstants.KEY_FORMAT));
}
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19008
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: goldfish
Gerrit-Change-Id: Ic0c12e32b85987cbc14d28e83fb2be644b922590
Gerrit-Change-Number: 19008
Gerrit-PatchSet: 1
Gerrit-Owner: Ayush Tripathi <[email protected]>
Gerrit-Reviewer: Murtadha Hubail <[email protected]>
Gerrit-Attention: Murtadha Hubail <[email protected]>
Gerrit-MessageType: newchange
