>From Hussain Towaileb <[email protected]>:
Hussain Towaileb has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21389?usp=email )
Change subject: [NO ISSUE][EXT]: fix parsing nulls and handling time micros for
avro
......................................................................
[NO ISSUE][EXT]: fix parsing nulls and handling time micros for avro
Details:
- Properly report Avro NULL value as null rather than missing and
omitting the whole field.
- Correctly only add the timezone offset once for TimeMicros type
for avro.
Ext-ref: MB-72320, MB-72680
Change-Id: I872be48898e01e7eefe4f1e2c85e72f1ca0c9c2e
---
M
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/avro/AvroLogicalTypesExampleGenerator.java
M
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/avro/avro-logical-types/avro-logical-types.01.ddl.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/avro/avro-logical-types/avro-logical-types.04.query.sqlpp
M
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/array-access/array-access.02.adm
M
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/array-access/array-access.03.adm
M
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/avro-logical-types/avro-logical-types.02.adm
M
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/avro-logical-types/avro-logical-types.03.adm
A
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/avro-logical-types/avro-logical-types.04.adm
M
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/select-all-fields/select-all-fields.2.adm
M
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/type-mismatch/type-mismatch.02.adm
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AvroDataParser.java
11 files changed, 109 insertions(+), 36 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/89/21389/1
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/avro/AvroLogicalTypesExampleGenerator.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/avro/AvroLogicalTypesExampleGenerator.java
index fd54627..0015c5b 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/avro/AvroLogicalTypesExampleGenerator.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/avro/AvroLogicalTypesExampleGenerator.java
@@ -29,6 +29,7 @@
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.io.DatumWriter;
import org.apache.avro.specific.SpecificDatumWriter;
+import org.apache.hyracks.util.annotations.AiProvenance;
import org.junit.Test;
public class AvroLogicalTypesExampleGenerator {
@@ -42,12 +43,15 @@
+ " { \"name\": \"timestampMillisField\", \"type\": { \"type\":
\"long\", \"logicalType\": \"timestamp-millis\" } },\n"
+ " { \"name\": \"timestampMicrosField\", \"type\": { \"type\":
\"long\", \"logicalType\": \"timestamp-micros\" } },\n"
+ " { \"name\": \"localTimestampMillisField\", \"type\": {
\"type\": \"long\", \"logicalType\": \"local-timestamp-millis\" } },\n"
- + " { \"name\": \"localTimestampMicrosField\", \"type\": {
\"type\": \"long\", \"logicalType\": \"local-timestamp-micros\" } }\n"
- + " ]\n" + "}";
+ + " { \"name\": \"localTimestampMicrosField\", \"type\": {
\"type\": \"long\", \"logicalType\": \"local-timestamp-micros\" } },\n"
+ + " { \"name\": \"nullableField\", \"type\": [\"null\",
\"string\"], \"default\": null }\n" + " ]\n"
+ + "}";
private static final String AVRO_GEN_BASEDIR =
"target/generated_avro_files";
private static final String FILE_NAME = "avro_logical_type.avro";
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_SONNET_4_6, tool =
AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind =
AiProvenance.ContributionKind.REFACTORED, notes = "Fix timeMicrosField test
value overflowing a day (was 12345678901234, > 24h in micros), which silently
overflowed int during Avro TimeMicros parsing")
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_SONNET_4_6, tool =
AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind =
AiProvenance.ContributionKind.REFACTORED, notes = "Add a null-valued nullable
field so tests can cover Avro NULL (not MISSING) parsing")
public static void writeLogicalTypesExample() throws IOException {
Schema schema = new Schema.Parser().parse(SCHEMA_STRING);
File destPath = new File(AVRO_GEN_BASEDIR);
@@ -66,11 +70,13 @@
record.put("uuidField", "123e4567-e89b-12d3-a456-426614174000");
record.put("dateField", 20061);
record.put("timeMillisField", 12345678);
- record.put("timeMicrosField", 12345678901234L);
+ // time-micros is microseconds after midnight (< 86,400,000,000);
45296789012 = 12:34:56.789012
+ record.put("timeMicrosField", 45296789012L);
record.put("timestampMillisField", 1733344079083L);
record.put("timestampMicrosField", 1733344079083000L);
record.put("localTimestampMillisField", 1733344079083L);
record.put("localTimestampMicrosField", 1733344079083000L);
+ record.put("nullableField", null);
dataFileWriter.append(record);
} catch (IOException e) {
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/avro/avro-logical-types/avro-logical-types.01.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/avro/avro-logical-types/avro-logical-types.01.ddl.sqlpp
index 73d8056..ca8294d 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/avro/avro-logical-types/avro-logical-types.01.ddl.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/avro/avro-logical-types/avro-logical-types.01.ddl.sqlpp
@@ -53,4 +53,19 @@
("time-to-long"="false"),
("timestamp-to-long"="false"),
("format" = "avro")
+);
+
+CREATE EXTERNAL DATASET AvroDataset3(AvroType) USING %adapter%
+(
+ %template%,
+ ("container"="playground"),
+ ("definition"="avro-data/reviews"),
+ ("include"="*avro_logical_type.avro"),
+ ("decimal-to-double"="true"),
+ ("uuid-to-string"="false"),
+ ("date-to-int"="false"),
+ ("time-to-long"="false"),
+ ("timestamp-to-long"="false"),
+ ("timezone"="PST"),
+ ("format" = "avro")
);
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/avro/avro-logical-types/avro-logical-types.04.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/avro/avro-logical-types/avro-logical-types.04.query.sqlpp
new file mode 100644
index 0000000..2ac7e29
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/avro/avro-logical-types/avro-logical-types.04.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+
+SELECT VALUE a
+FROM AvroDataset3 a;
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/array-access/array-access.02.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/array-access/array-access.02.adm
index 18f3275..ff1820c 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/array-access/array-access.02.adm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/array-access/array-access.02.adm
@@ -1,2 +1,2 @@
{ "display_url": "string" }
-{ }
+{ "display_url": null }
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/array-access/array-access.03.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/array-access/array-access.03.adm
index 695240b..1139820 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/array-access/array-access.03.adm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/array-access/array-access.03.adm
@@ -1 +1,2 @@
{ "display_url": [ "string" ] }
+{ "display_url": null }
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/avro-logical-types/avro-logical-types.02.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/avro-logical-types/avro-logical-types.02.adm
index 7690b33..6d2a06c 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/avro-logical-types/avro-logical-types.02.adm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/avro-logical-types/avro-logical-types.02.adm
@@ -1 +1 @@
-{ "decimalField": 2.58, "uuidField": "123e4567-e89b-12d3-a456-426614174000",
"dateField": 20061, "timeMillisField": 12345678, "timeMicrosField": -539222987,
"timestampMillisField": 1733344079083, "timestampMicrosField": 1733344079083,
"localTimestampMillisField": 1733344079083, "localTimestampMicrosField":
1733344079083 }
\ No newline at end of file
+{ "decimalField": 2.58, "uuidField": "123e4567-e89b-12d3-a456-426614174000",
"dateField": 20061, "timeMillisField": 12345678, "timeMicrosField": 45296789,
"timestampMillisField": 1733344079083, "timestampMicrosField": 1733344079083,
"localTimestampMillisField": 1733344079083, "localTimestampMicrosField":
1733344079083, "nullableField": null }
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/avro-logical-types/avro-logical-types.03.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/avro-logical-types/avro-logical-types.03.adm
index 51eedae..b01df9c 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/avro-logical-types/avro-logical-types.03.adm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/avro-logical-types/avro-logical-types.03.adm
@@ -1 +1 @@
-{ "decimalField": 2.58, "uuidField":
uuid("123e4567-e89b-12d3-a456-426614174000"), "dateField": date("2024-12-04"),
"timeMillisField": time("03:25:45.678"), "timeMicrosField":
time("18:12:57.013"), "timestampMillisField":
datetime("2024-12-04T20:27:59.083"), "timestampMicrosField":
datetime("2024-12-04T20:27:59.083"), "localTimestampMillisField":
datetime("2024-12-04T20:27:59.083"), "localTimestampMicrosField":
datetime("2024-12-04T20:27:59.083") }
\ No newline at end of file
+{ "decimalField": 2.58, "uuidField":
uuid("123e4567-e89b-12d3-a456-426614174000"), "dateField": date("2024-12-04"),
"timeMillisField": time("03:25:45.678"), "timeMicrosField":
time("12:34:56.789"), "timestampMillisField":
datetime("2024-12-04T20:27:59.083"), "timestampMicrosField":
datetime("2024-12-04T20:27:59.083"), "localTimestampMillisField":
datetime("2024-12-04T20:27:59.083"), "localTimestampMicrosField":
datetime("2024-12-04T20:27:59.083"), "nullableField": null }
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/avro-logical-types/avro-logical-types.04.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/avro-logical-types/avro-logical-types.04.adm
new file mode 100644
index 0000000..dbfb24c
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/avro-logical-types/avro-logical-types.04.adm
@@ -0,0 +1 @@
+{ "decimalField": 2.58, "uuidField":
uuid("123e4567-e89b-12d3-a456-426614174000"), "dateField": date("2024-12-04"),
"timeMillisField": time("03:25:45.678"), "timeMicrosField":
time("12:34:56.789"), "timestampMillisField":
datetime("2024-12-04T12:27:59.083"), "timestampMicrosField":
datetime("2024-12-04T12:27:59.083"), "localTimestampMillisField":
datetime("2024-12-04T20:27:59.083"), "localTimestampMicrosField":
datetime("2024-12-04T20:27:59.083"), "nullableField": null }
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/select-all-fields/select-all-fields.2.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/select-all-fields/select-all-fields.2.adm
index 53f2518..8781189 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/select-all-fields/select-all-fields.2.adm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/select-all-fields/select-all-fields.2.adm
@@ -1,2 +1,2 @@
{ "coordinates": { "coordinates": [ 1.1 ], "type": "string" }, "created_at":
"string", "entities": { "urls": [ { "display_url": "string", "expanded_url":
"string", "indices": [ 1 ], "url": "string" } ], "user_mentions": [ { "id": 1,
"id_str": "string", "indices": [ 1 ], "name": "string", "screen_name": "string"
} ] }, "favorite_count": 1, "favorited": true, "filter_level": "string", "geo":
{ "coordinates": [ 1.1 ], "type": "string" }, "id": "0000000", "id_str":
"string", "in_reply_to_screen_name": "string", "in_reply_to_status_id": 1,
"in_reply_to_status_id_str": "string", "in_reply_to_user_id": 1,
"in_reply_to_user_id_str": "string", "is_quote_status": true, "lang": "string",
"place": { "bounding_box": { "coordinates": [ [ [ 1.1 ] ] ], "type": "string"
}, "country": "string", "country_code": "string", "full_name": "string", "id":
"string", "name": "string", "place_type": "string", "url": "string" },
"possibly_sensitive": true, "quoted_status": { "created_at": "string",
"entities": { "user_mentions": [ { "id": 1, "id_str": "string", "indices": [ 1
], "name": "string", "screen_name": "string" } ] }, "favorite_count": 1,
"favorited": true, "filter_level": "string", "id": 1, "id_str": "string",
"in_reply_to_screen_name": "string", "in_reply_to_status_id": 1,
"in_reply_to_status_id_str": "string", "in_reply_to_user_id": 1,
"in_reply_to_user_id_str": "string", "is_quote_status": true, "lang": "string",
"retweet_count": 1, "retweeted": true, "source": "string", "text": "string",
"truncated": true, "user": { "contributors_enabled": true, "created_at":
"string", "default_profile": true, "default_profile_image": true,
"description": "string", "favourites_count": 1, "followers_count": 1,
"friends_count": 1, "geo_enabled": true, "id": 1, "id_str": "string",
"is_translator": true, "lang": "string", "listed_count": 1, "name": "string",
"profile_background_color": "string", "profile_background_image_url": "string",
"profile_background_image_url_https": "string", "profile_background_tile":
true, "profile_banner_url": "string", "profile_image_url": "string",
"profile_image_url_https": "string", "profile_link_color": "string",
"profile_sidebar_border_color": "string", "profile_sidebar_fill_color":
"string", "profile_text_color": "string", "profile_use_background_image": true,
"protected": true, "screen_name": "string", "statuses_count": 1, "verified":
true } }, "quoted_status_id": 1, "quoted_status_id_str": "string",
"retweet_count": 1, "retweeted": true, "source": "string", "text": "string",
"timestamp_ms": "string", "truncated": true, "user": { "contributors_enabled":
true, "created_at": "string", "default_profile": true, "default_profile_image":
true, "description": "string", "favourites_count": 1, "followers_count": 1,
"friends_count": 1, "geo_enabled": true, "id": 1, "id_str": "string",
"is_translator": true, "lang": "string", "listed_count": 1, "location":
"string", "name": "string", "profile_background_color": "string",
"profile_background_image_url": "string", "profile_background_image_url_https":
"string", "profile_background_tile": true, "profile_banner_url": "string",
"profile_image_url": "string", "profile_image_url_https": "string",
"profile_link_color": "string", "profile_sidebar_border_color": "string",
"profile_sidebar_fill_color": "string", "profile_text_color": "string",
"profile_use_background_image": true, "protected": true, "screen_name":
"string", "statuses_count": 1, "time_zone": "string", "url": "string",
"utc_offset": 1, "verified": true } }
-{ "coordinates": { "coordinates": [ 1.1 ], "type": "string" }, "created_at":
"string", "favorite_count": 1, "favorited": true, "filter_level": "string",
"geo": { "coordinates": [ 1.1 ], "type": "string" }, "id":
"11111111111111111111", "id_str": "string", "in_reply_to_screen_name":
"string", "in_reply_to_status_id": 1, "in_reply_to_status_id_str": "string",
"in_reply_to_user_id": 1, "in_reply_to_user_id_str": "string",
"is_quote_status": true, "lang": "string", "place": { "bounding_box": {
"coordinates": [ [ [ 1.1 ] ] ], "type": "string" }, "country": "string",
"country_code": "string", "full_name": "string", "id": "string", "name":
"string", "place_type": "string", "url": "string" }, "possibly_sensitive":
true, "quoted_status": { "created_at": "string", "entities": { "user_mentions":
[ { "id": 1, "id_str": "string", "indices": [ 1 ], "name": "string",
"screen_name": "string" } ] }, "favorite_count": 1, "favorited": true,
"filter_level": "string", "id": 1, "id_str": "string",
"in_reply_to_screen_name": "string", "in_reply_to_status_id": 1,
"in_reply_to_status_id_str": "string", "in_reply_to_user_id": 1,
"in_reply_to_user_id_str": "string", "is_quote_status": true, "lang": "string",
"retweet_count": 1, "retweeted": true, "source": "string", "text": "string",
"truncated": true, "user": { "contributors_enabled": true, "created_at":
"string", "default_profile": true, "default_profile_image": true,
"description": "string", "favourites_count": 1, "followers_count": 1,
"friends_count": 1, "geo_enabled": true, "id": 1, "id_str": "string",
"is_translator": true, "lang": "string", "listed_count": 1, "name": "string",
"profile_background_color": "string", "profile_background_image_url": "string",
"profile_background_image_url_https": "string", "profile_background_tile":
true, "profile_banner_url": "string", "profile_image_url": "string",
"profile_image_url_https": "string", "profile_link_color": "string",
"profile_sidebar_border_color": "string", "profile_sidebar_fill_color":
"string", "profile_text_color": "string", "profile_use_background_image": true,
"protected": true, "screen_name": "string", "statuses_count": 1, "verified":
true } }, "quoted_status_id": 1, "quoted_status_id_str": "string",
"retweet_count": 1, "retweeted": true, "source": "string", "text": "string",
"timestamp_ms": "string", "truncated": true, "user": { "contributors_enabled":
true, "created_at": "string", "default_profile": true, "default_profile_image":
true, "description": "string", "favourites_count": 1, "followers_count": 1,
"friends_count": 1, "geo_enabled": true, "id": 1, "id_str": "string",
"is_translator": true, "lang": "string", "listed_count": 1, "location":
"string", "name": "string", "profile_background_color": "string",
"profile_background_image_url": "string", "profile_background_image_url_https":
"string", "profile_background_tile": true, "profile_banner_url": "string",
"profile_image_url": "string", "profile_image_url_https": "string",
"profile_link_color": "string", "profile_sidebar_border_color": "string",
"profile_sidebar_fill_color": "string", "profile_text_color": "string",
"profile_use_background_image": true, "protected": true, "screen_name":
"string", "statuses_count": 1, "time_zone": "string", "url": "string",
"utc_offset": 1, "verified": true } }
+{ "coordinates": { "coordinates": [ 1.1 ], "type": "string" }, "created_at":
"string", "entities": null, "favorite_count": 1, "favorited": true,
"filter_level": "string", "geo": { "coordinates": [ 1.1 ], "type": "string" },
"id": "11111111111111111111", "id_str": "string", "in_reply_to_screen_name":
"string", "in_reply_to_status_id": 1, "in_reply_to_status_id_str": "string",
"in_reply_to_user_id": 1, "in_reply_to_user_id_str": "string",
"is_quote_status": true, "lang": "string", "place": { "bounding_box": {
"coordinates": [ [ [ 1.1 ] ] ], "type": "string" }, "country": "string",
"country_code": "string", "full_name": "string", "id": "string", "name":
"string", "place_type": "string", "url": "string" }, "possibly_sensitive":
true, "quoted_status": { "created_at": "string", "entities": { "user_mentions":
[ { "id": 1, "id_str": "string", "indices": [ 1 ], "name": "string",
"screen_name": "string" } ] }, "favorite_count": 1, "favorited": true,
"filter_level": "string", "id": 1, "id_str": "string",
"in_reply_to_screen_name": "string", "in_reply_to_status_id": 1,
"in_reply_to_status_id_str": "string", "in_reply_to_user_id": 1,
"in_reply_to_user_id_str": "string", "is_quote_status": true, "lang": "string",
"retweet_count": 1, "retweeted": true, "source": "string", "text": "string",
"truncated": true, "user": { "contributors_enabled": true, "created_at":
"string", "default_profile": true, "default_profile_image": true,
"description": "string", "favourites_count": 1, "followers_count": 1,
"friends_count": 1, "geo_enabled": true, "id": 1, "id_str": "string",
"is_translator": true, "lang": "string", "listed_count": 1, "name": "string",
"profile_background_color": "string", "profile_background_image_url": "string",
"profile_background_image_url_https": "string", "profile_background_tile":
true, "profile_banner_url": "string", "profile_image_url": "string",
"profile_image_url_https": "string", "profile_link_color": "string",
"profile_sidebar_border_color": "string", "profile_sidebar_fill_color":
"string", "profile_text_color": "string", "profile_use_background_image": true,
"protected": true, "screen_name": "string", "statuses_count": 1, "verified":
true } }, "quoted_status_id": 1, "quoted_status_id_str": "string",
"retweet_count": 1, "retweeted": true, "source": "string", "text": "string",
"timestamp_ms": "string", "truncated": true, "user": { "contributors_enabled":
true, "created_at": "string", "default_profile": true, "default_profile_image":
true, "description": "string", "favourites_count": 1, "followers_count": 1,
"friends_count": 1, "geo_enabled": true, "id": 1, "id_str": "string",
"is_translator": true, "lang": "string", "listed_count": 1, "location":
"string", "name": "string", "profile_background_color": "string",
"profile_background_image_url": "string", "profile_background_image_url_https":
"string", "profile_background_tile": true, "profile_banner_url": "string",
"profile_image_url": "string", "profile_image_url_https": "string",
"profile_link_color": "string", "profile_sidebar_border_color": "string",
"profile_sidebar_fill_color": "string", "profile_text_color": "string",
"profile_use_background_image": true, "protected": true, "screen_name":
"string", "statuses_count": 1, "time_zone": "string", "url": "string",
"utc_offset": 1, "verified": true } }
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/type-mismatch/type-mismatch.02.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/type-mismatch/type-mismatch.02.adm
index 0be5d98..d252328 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/type-mismatch/type-mismatch.02.adm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/type-mismatch/type-mismatch.02.adm
@@ -1,2 +1,2 @@
true
-true
\ No newline at end of file
+false
\ No newline at end of file
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AvroDataParser.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AvroDataParser.java
index faf6d72..e219781 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AvroDataParser.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AvroDataParser.java
@@ -55,6 +55,7 @@
import org.apache.hyracks.api.exceptions.Warning;
import org.apache.hyracks.data.std.api.IMutableValueStorage;
import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.util.annotations.AiProvenance;
public class AvroDataParser extends AbstractDataParser implements
IRecordDataParser<GenericRecord> {
private final AvroConverterContext parserContext;
@@ -88,17 +89,16 @@
Object fieldValue = record.get(fieldName);
ATypeTag typeTag = getTypeTag(fieldSchema, fieldValue);
- IValueReference value = null;
+ IValueReference value;
if (valueEmbedder.shouldEmbed(fieldName, typeTag)) {
value = valueEmbedder.getEmbeddedValue();
- } else if (fieldValue != null) {
+ } else {
valueBuffer.reset();
parseValue(fieldSchema, fieldValue,
valueBuffer.getDataOutput());
value = valueBuffer;
}
- if (value != null) {
- // Ignore missing values
+ if (value != null || typeTag == ATypeTag.NULL) {
objectBuilder.addField(parserContext.getSerializedFieldName(fieldName), value);
}
}
@@ -148,26 +148,30 @@
parserContext.exitCollection(item, listBuilder);
}
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool =
AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind =
AiProvenance.ContributionKind.REFACTORED, notes = "Throw a well-formed
unsupported-type error instead of a 1-arg message that crashes formatting")
private void parseUnion(Schema unionSchema, Object value, DataOutput out)
throws IOException {
Schema actualSchema = getActualSchema(unionSchema, value);
if (actualSchema != null) {
parseValue(actualSchema, value, out);
} else {
- throw new RuntimeDataException(ErrorCode.TYPE_UNSUPPORTED,
unionSchema.getType());
+ throw createUnsupportedException(unionSchema);
}
}
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_SONNET_4_6, tool =
AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind =
AiProvenance.ContributionKind.REFACTORED, notes = "Return null-typed schema for
null values in nullable unions instead of returning null")
private Schema getActualSchema(Schema unionSchema, Object value) {
List<Schema> possibleTypes = unionSchema.getTypes();
+ Schema nullSchema = null;
for (Schema possibleType : possibleTypes) {
Schema.Type schemaType = possibleType.getType();
- if (schemaType != NULL) {
- if (matchesType(value, schemaType)) {
- return possibleType;
- }
+ if (schemaType == NULL) {
+ nullSchema = possibleType;
+ } else if (matchesType(value, schemaType)) {
+ return possibleType;
}
}
- return null;
+ // value is null — return the null branch of the union if present
+ return value == null ? nullSchema : null;
}
private boolean matchesType(Object value, Schema.Type schemaType) {
@@ -197,6 +201,7 @@
}
}
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool =
AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind =
AiProvenance.ContributionKind.REFACTORED, notes = "Map null values to NULL
instead of MISSING; make union fall-through explicit")
private ATypeTag getTypeTag(Schema schema, Object value) throws
HyracksDataException {
Schema.Type schemaType = schema.getType();
LogicalType logicalType = schema.getLogicalType();
@@ -235,8 +240,9 @@
}
if (value == null) {
- // The 'value' is missing
- return ATypeTag.MISSING;
+ // A null value (e.g. the null branch of a nullable union) maps to
NULL, not MISSING.
+ // Avro records always carry every declared field, so there is no
true "missing" here.
+ return ATypeTag.NULL;
}
switch (schemaType) {
@@ -264,12 +270,15 @@
if (actualSchema != null) {
return getTypeTag(actualSchema, value);
}
+ throw createUnsupportedException(schema);
default:
throw createUnsupportedException(schema);
-
}
}
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool =
AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind =
AiProvenance.ContributionKind.REFACTORED, notes = "Fix TimeMicros applying the
timezone offset twice")
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_SONNET_4_6, tool =
AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind =
AiProvenance.ContributionKind.REFACTORED, notes = "Stop applying the timezone
offset to local-timestamp-millis/micros, which are already wall-clock values
with no associated timezone")
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_SONNET_4_6, tool =
AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind =
AiProvenance.ContributionKind.REFACTORED, notes = "Stop applying the timezone
offset to time-millis/time-micros, which have no associated timezone (only
timestamp-millis/micros are UTC instants)")
private void parseLogicalValue(LogicalType logicalType, Object value,
DataOutput out) throws IOException {
if (logicalType instanceof LogicalTypes.Uuid) {
if (parserContext.isUuidAsString()) {
@@ -288,25 +297,24 @@
parserContext.serializeDate(value, out);
}
} else if (logicalType instanceof LogicalTypes.TimeMicros) {
+ // time-micros is a time of day with no reference to any timezone;
it must not be shifted
+ // by the configured offset (unlike timestamp-millis/micros, which
are UTC instants).
int timeInMillis = (int) TimeUnit.MICROSECONDS.toMillis(((Number)
value).longValue());
- int offset = parserContext.getTimeZoneOffset();
- timeInMillis = timeInMillis + offset;
- if (parserContext.isTimeAsLong()) {
- serializeLong(timeInMillis, out);
- } else {
- parserContext.serializeTime(timeInMillis + offset, out);
- }
- } else if (logicalType instanceof LogicalTypes.TimeMillis) {
- int timeInMillis = ((Number) value).intValue();
- int offset = parserContext.getTimeZoneOffset();
- timeInMillis = timeInMillis + offset;
if (parserContext.isTimeAsLong()) {
serializeLong(timeInMillis, out);
} else {
parserContext.serializeTime(timeInMillis, out);
}
- } else if (logicalType instanceof LogicalTypes.TimestampMicros
- || logicalType instanceof LogicalTypes.LocalTimestampMicros) {
+ } else if (logicalType instanceof LogicalTypes.TimeMillis) {
+ // time-millis is a time of day with no reference to any timezone;
it must not be shifted
+ // by the configured offset (unlike timestamp-millis/micros, which
are UTC instants).
+ int timeInMillis = ((Number) value).intValue();
+ if (parserContext.isTimeAsLong()) {
+ serializeLong(timeInMillis, out);
+ } else {
+ parserContext.serializeTime(timeInMillis, out);
+ }
+ } else if (logicalType instanceof LogicalTypes.TimestampMicros) {
long timeStampInMicros = ((Number) value).longValue();
int offset = parserContext.getTimeZoneOffset();
long timeStampInMillis =
TimeUnit.MICROSECONDS.toMillis(timeStampInMicros);
@@ -316,8 +324,7 @@
} else {
parserContext.serializeDateTime(timeStampInMillis, out);
}
- } else if (logicalType instanceof LogicalTypes.TimestampMillis
- || logicalType instanceof LogicalTypes.LocalTimestampMillis) {
+ } else if (logicalType instanceof LogicalTypes.TimestampMillis) {
long timeStampInMillis = ((Number) value).longValue();
int offset = parserContext.getTimeZoneOffset();
timeStampInMillis = timeStampInMillis + offset;
@@ -326,6 +333,25 @@
} else {
parserContext.serializeDateTime(timeStampInMillis, out);
}
+ } else if (logicalType instanceof LogicalTypes.LocalTimestampMicros) {
+ // local-timestamp-micros is already wall-clock time with no
associated timezone;
+ // unlike timestamp-micros (a UTC instant), it must not be shifted
by the configured offset.
+ long timeStampInMicros = ((Number) value).longValue();
+ long timeStampInMillis =
TimeUnit.MICROSECONDS.toMillis(timeStampInMicros);
+ if (parserContext.isTimestampAsLong()) {
+ serializeLong(timeStampInMillis, out);
+ } else {
+ parserContext.serializeDateTime(timeStampInMillis, out);
+ }
+ } else if (logicalType instanceof LogicalTypes.LocalTimestampMillis) {
+ // local-timestamp-millis is already wall-clock time with no
associated timezone;
+ // unlike timestamp-millis (a UTC instant), it must not be shifted
by the configured offset.
+ long timeStampInMillis = ((Number) value).longValue();
+ if (parserContext.isTimestampAsLong()) {
+ serializeLong(timeStampInMillis, out);
+ } else {
+ parserContext.serializeDateTime(timeStampInMillis, out);
+ }
} else {
throw createUnsupportedException(logicalType.getName());
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21389?usp=email
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings?usp=email
Gerrit-MessageType: newchange
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: I872be48898e01e7eefe4f1e2c85e72f1ca0c9c2e
Gerrit-Change-Number: 21389
Gerrit-PatchSet: 1
Gerrit-Owner: Hussain Towaileb <[email protected]>
