github-actions[bot] commented on code in PR #66302:
URL: https://github.com/apache/doris/pull/66302#discussion_r3705702267
##########
be/src/format_v2/parquet/native_schema_desc.cpp:
##########
@@ -55,6 +57,412 @@ static bool is_map_node(const tparquet::SchemaElement&
schema) {
(schema.__isset.logicalType && schema.logicalType.__isset.MAP);
}
+static bool is_variant_node(const tparquet::SchemaElement& schema) {
+ return schema.__isset.logicalType && schema.logicalType.__isset.VARIANT;
+}
+
+enum class VariantPrimitiveAnnotation : uint8_t {
+ NONE,
+ INT8,
+ INT16,
+ DECIMAL,
+ DATE,
+ TIME_MICROS,
+ TIMESTAMP_MICROS,
+ TIMESTAMP_NANOS,
+ STRING,
+ UUID,
+ UNSUPPORTED,
+};
+
+static VariantPrimitiveAnnotation variant_logical_annotation(
+ const tparquet::SchemaElement& schema) {
+ if (!schema.__isset.logicalType) {
+ return VariantPrimitiveAnnotation::NONE;
+ }
+ const auto& logical = schema.logicalType;
+ if (logical.__isset.INTEGER) {
+ if (!logical.INTEGER.isSigned) {
+ return VariantPrimitiveAnnotation::UNSUPPORTED;
+ }
+ if (logical.INTEGER.bitWidth == 8) {
+ return VariantPrimitiveAnnotation::INT8;
+ }
+ if (logical.INTEGER.bitWidth == 16) {
+ return VariantPrimitiveAnnotation::INT16;
+ }
+ return VariantPrimitiveAnnotation::UNSUPPORTED;
+ }
+ if (logical.__isset.DECIMAL) {
+ return VariantPrimitiveAnnotation::DECIMAL;
+ }
+ if (logical.__isset.DATE) {
+ return VariantPrimitiveAnnotation::DATE;
+ }
+ if (logical.__isset.TIME) {
+ return !logical.TIME.isAdjustedToUTC &&
logical.TIME.unit.__isset.MICROS
+ ? VariantPrimitiveAnnotation::TIME_MICROS
+ : VariantPrimitiveAnnotation::UNSUPPORTED;
+ }
+ if (logical.__isset.TIMESTAMP) {
+ if (logical.TIMESTAMP.unit.__isset.MICROS) {
+ return VariantPrimitiveAnnotation::TIMESTAMP_MICROS;
+ }
+ if (logical.TIMESTAMP.unit.__isset.NANOS) {
+ return VariantPrimitiveAnnotation::TIMESTAMP_NANOS;
+ }
+ return VariantPrimitiveAnnotation::UNSUPPORTED;
+ }
+ if (logical.__isset.STRING) {
+ return VariantPrimitiveAnnotation::STRING;
+ }
+ if (logical.__isset.UUID) {
+ return VariantPrimitiveAnnotation::UUID;
+ }
+ const bool empty = !logical.__isset.MAP && !logical.__isset.LIST &&
!logical.__isset.ENUM &&
+ !logical.__isset.UNKNOWN && !logical.__isset.JSON &&
!logical.__isset.BSON &&
+ !logical.__isset.FLOAT16 && !logical.__isset.GEOMETRY &&
+ !logical.__isset.GEOGRAPHY && !logical.__isset.VARIANT;
+ return empty ? VariantPrimitiveAnnotation::NONE :
VariantPrimitiveAnnotation::UNSUPPORTED;
+}
+
+static VariantPrimitiveAnnotation variant_converted_annotation(
+ const tparquet::SchemaElement& schema) {
+ if (!schema.__isset.converted_type) {
+ return VariantPrimitiveAnnotation::NONE;
+ }
+ switch (schema.converted_type) {
+ case tparquet::ConvertedType::INT_8:
+ return VariantPrimitiveAnnotation::INT8;
+ case tparquet::ConvertedType::INT_16:
+ return VariantPrimitiveAnnotation::INT16;
+ case tparquet::ConvertedType::DECIMAL:
+ return VariantPrimitiveAnnotation::DECIMAL;
+ case tparquet::ConvertedType::DATE:
+ return VariantPrimitiveAnnotation::DATE;
+ case tparquet::ConvertedType::TIME_MICROS:
+ return VariantPrimitiveAnnotation::TIME_MICROS;
+ case tparquet::ConvertedType::TIMESTAMP_MICROS:
+ return VariantPrimitiveAnnotation::TIMESTAMP_MICROS;
Review Comment:
[P1] Preserve the UTC instant for legacy shredded timestamps
This accepts legacy `ConvertedType::TIMESTAMP_MICROS`, whose Parquet
semantics are UTC-adjusted, but the Variant-scoped
`_enable_mapping_timestamp_tz` only affects logical-type conversion.
Converted-type conversion still produces `DATETIMEV2`, so native decode first
renders epoch micros in the session time zone; `append_typed_scalar()` then
converts those civil fields back to micros while tagging the value
UTC-adjusted. In Asia/Shanghai, epoch 0 is therefore re-encoded as +8 hours,
making the Variant value session-zone-dependent and different from equivalent
`TIMESTAMP(true, MICROS)` input. Please map this accepted legacy annotation to
`TIMESTAMPTZ` inside Variant (or preserve the raw epoch) and cover a non-UTC
session.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
