github-actions[bot] commented on code in PR #66761: URL: https://github.com/apache/doris/pull/66761#discussion_r3853619667
########## be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp: ########## @@ -0,0 +1,360 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "core/data_type_serde/data_type_timestamp_ns_serde.h" + +#include <arrow/builder.h> +#include <cctz/time_zone.h> + +#include <algorithm> +#include <cctype> +#include <limits> +#include <string> + +#include "common/config.h" +#include "common/exception.h" +#include "core/assert_cast.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/data_type_serde/arrow_validation.h" +#include "core/value/vdatetime_value.h" +#include "exprs/function/cast/cast_to_datetimev2_impl.hpp" +#include "util/mysql_row_buffer.h" +#include "util/unaligned.h" + +namespace doris { + +namespace { + +Status get_nanos_per_arrow_timestamp_unit(arrow::TimeUnit::type unit, int64_t* nanos_per_unit) { + switch (unit) { + case arrow::TimeUnit::SECOND: + *nanos_per_unit = TimeStampNsValue::NANOS_PER_SECOND; + break; + case arrow::TimeUnit::MILLI: + *nanos_per_unit = TimeStampNsValue::NANOS_PER_MILLISECOND; + break; + case arrow::TimeUnit::MICRO: + *nanos_per_unit = TimeStampNsValue::NANOS_PER_MICROSECOND; + break; + case arrow::TimeUnit::NANO: + *nanos_per_unit = 1; + break; + default: + return Status::InvalidArgument("Unsupported Arrow timestamp unit: {}", + static_cast<int>(unit)); + } + return Status::OK(); +} + +} // namespace + +Status parse_timestamp_ns(StringRef str, int64_t* epoch_nanos, + const cctz::time_zone* local_time_zone) { + std::string input(str.data, str.size); + const size_t dot = input.rfind('.'); + size_t fraction_begin = std::string::npos; + size_t fraction_end = std::string::npos; + if (dot != std::string::npos && dot + 1 < input.size() && + std::isdigit(static_cast<unsigned char>(input[dot + 1]))) { + fraction_begin = dot + 1; + fraction_end = fraction_begin; + while (fraction_end < input.size() && + std::isdigit(static_cast<unsigned char>(input[fraction_end]))) { + ++fraction_end; + } + } + + uint32_t nanos = 0; + size_t fraction_length = 0; + if (fraction_begin != std::string::npos) { + fraction_length = fraction_end - fraction_begin; + const size_t copied_digits = + std::min<size_t>(fraction_length, TimeStampNsValue::FRACTIONAL_DIGITS); + for (size_t i = 0; i < copied_digits; ++i) { + nanos = nanos * 10 + static_cast<uint32_t>(input[fraction_begin + i] - '0'); + } + for (size_t i = copied_digits; i < TimeStampNsValue::FRACTIONAL_DIGITS; ++i) { + nanos *= 10; + } + } + + if (fraction_length > TimeStampNsValue::FRACTIONAL_DIGITS && + input[fraction_begin + TimeStampNsValue::FRACTIONAL_DIGITS] >= '5') { + ++nanos; + } + + const bool carry_second = nanos == TimeStampNsValue::NANOS_PER_SECOND; + + // Keep the fractional token in place so that the legacy parser validates its position and all + // trailing syntax. Its scale-0 rounding happens before an explicit source timezone is converted + // to the session timezone. Mark only a nanosecond carry for that existing path, so a DST + // transition is crossed on the source instant timeline rather than on the local civil clock. + if (fraction_begin != std::string::npos) { + std::fill(input.begin() + fraction_begin, input.begin() + fraction_end, '0'); + if (carry_second) { + input[fraction_begin] = '5'; + } + } + const StringRef input_ref(input.data(), input.size()); + DateV2Value<DateTimeV2ValueType> datetime; + CastParameters params {.status = Status::OK(), .is_strict = true}; + CastToDatetimeV2::from_string_strict_mode<DatelikeParseMode::STRICT>( + input_ref, datetime, local_time_zone, 0, params); + if (!params.status.ok()) { + if (dot != std::string::npos) { + return Status::InvalidArgument("Invalid TIMESTAMP_NS value '{}'", + std::string(str.data, str.size)); + } + return params.status; + } + + if (carry_second) { + nanos = 0; + } + datetime.set_microsecond(nanos / TimeStampNsValue::NANOS_PER_MICROSECOND); + TimeStampNsValue value; + if (!value.from_datetime( + datetime, static_cast<uint16_t>(nanos % TimeStampNsValue::NANOS_PER_MICROSECOND))) { + return Status::InvalidArgument( + "TIMESTAMP_NS value '{}' is outside [{}, {}]", std::string(str.data, str.size), + TimeStampNsValue(std::numeric_limits<int64_t>::min()).to_string(), + TimeStampNsValue(std::numeric_limits<int64_t>::max()).to_string()); + } + *epoch_nanos = value.epoch_nanos(); + return Status::OK(); +} + +Status DataTypeTimeStampNsSerDe::from_string_batch(const ColumnString& strings, + ColumnNullable& result, + const FormatOptions& options) const { + auto& data = assert_cast<ColumnTimeStampNs&>(result.get_nested_column()).get_data(); + auto& null_map = result.get_null_map_column().get_data(); + result.resize(strings.size()); + for (size_t i = 0; i < strings.size(); ++i) { + int64_t value = 0; + const auto status = parse_timestamp_ns(strings.get_data_at(i), &value, options.timezone); + null_map[i] = !status.ok(); + data[i] = TimeStampNsValue(value); + } + return Status::OK(); +} + +Status DataTypeTimeStampNsSerDe::from_string_strict_mode_batch( + const ColumnString& strings, IColumn& result, const FormatOptions& options, + const NullMap::value_type* null_map) const { + auto& data = assert_cast<ColumnTimeStampNs&>(result).get_data(); + result.resize(strings.size()); + for (size_t i = 0; i < strings.size(); ++i) { + if (null_map != nullptr && null_map[i]) { + continue; + } + int64_t value = 0; + RETURN_IF_ERROR(parse_timestamp_ns(strings.get_data_at(i), &value, options.timezone)); + data[i] = TimeStampNsValue(value); + } + return Status::OK(); +} + +Status DataTypeTimeStampNsSerDe::from_string(StringRef& str, IColumn& column, + const FormatOptions& options) const { + int64_t value = 0; + RETURN_IF_ERROR(parse_timestamp_ns(str, &value, options.timezone)); + assert_cast<ColumnTimeStampNs&>(column).insert_value(TimeStampNsValue(value)); + return Status::OK(); +} + +Status DataTypeTimeStampNsSerDe::from_string_strict_mode(StringRef& str, IColumn& column, + const FormatOptions& options) const { + return from_string(str, column, options); +} + +Status DataTypeTimeStampNsSerDe::serialize_column_to_json(const IColumn& column, int64_t start_idx, + int64_t end_idx, BufferWritable& bw, + FormatOptions& options) const { + SERIALIZE_COLUMN_TO_JSON(); +} + +Status DataTypeTimeStampNsSerDe::serialize_one_cell_to_json(const IColumn& column, int64_t row_num, + BufferWritable& bw, + FormatOptions& options) const { + auto [column_ptr, index] = check_column_const_set_readability(column, row_num); + if (_nesting_level > 1) { + bw.write('"'); + } + const auto value = + assert_cast<const ColumnTimeStampNs&, TypeCheckOnRelease::DISABLE>(*column_ptr) + .get_element(index); + const std::string result = value.to_string(); + bw.write(result.data(), result.size()); + if (_nesting_level > 1) { + bw.write('"'); + } + return Status::OK(); +} + +Status DataTypeTimeStampNsSerDe::deserialize_column_from_json_vector( + IColumn& column, std::vector<Slice>& slices, uint64_t* num_deserialized, + const FormatOptions& options) const { + DESERIALIZE_COLUMN_FROM_JSON_VECTOR(); + return Status::OK(); +} + +Status DataTypeTimeStampNsSerDe::deserialize_one_cell_from_json( + IColumn& column, Slice& slice, const FormatOptions& options) const { + if (_nesting_level > 1) { + slice.trim_quote(); + } + StringRef str(slice.data, slice.size); + return from_string(str, column, options); +} + +Status DataTypeTimeStampNsSerDe::deserialize_column_from_jsonb(IColumn& column, + const JsonbValue* jsonb_value, + CastParameters& cast_params) const { + DORIS_CHECK(jsonb_value->isString()); + return parse_column_from_jsonb_string(column, jsonb_value, cast_params); +} + +Status DataTypeTimeStampNsSerDe::deserialize_column_from_jsonb_vector( + ColumnNullable& column_to, const ColumnString& column_from, + CastParameters& cast_params) const { + return DataTypeSerDe::deserialize_column_from_jsonb_vector(column_to, column_from, cast_params); +} + +Status DataTypeTimeStampNsSerDe::write_column_to_arrow(const IColumn& column, + const NullMap* null_map, + arrow::ArrayBuilder* array_builder, + int64_t start, int64_t end, + const cctz::time_zone&) const { + const auto& data = assert_cast<const ColumnTimeStampNs&>(column).get_data(); + auto& builder = assert_cast<arrow::TimestampBuilder&>(*array_builder); + const auto timestamp_type = + std::static_pointer_cast<arrow::TimestampType>(array_builder->type()); + int64_t nanos_per_unit = 0; + RETURN_IF_ERROR(get_nanos_per_arrow_timestamp_unit(timestamp_type->unit(), &nanos_per_unit)); + + for (int64_t i = start; i < end; ++i) { + if (null_map != nullptr && (*null_map)[i]) { + RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(), column, builder)); + continue; + } + + const int64_t epoch_nanos = data[i].epoch_nanos(); + // The default Arrow schema uses NANO. If a caller supplies a coarser timestamp schema, + // accept it only when conversion is exact instead of silently discarding nanoseconds. + if (epoch_nanos % nanos_per_unit != 0) { + return Status::InvalidArgument( + "TIMESTAMP_NS value {} cannot be represented exactly as Arrow timestamp unit " + "{}", + epoch_nanos, static_cast<int>(timestamp_type->unit())); + } + RETURN_IF_ERROR( + checkArrowStatus(builder.Append(epoch_nanos / nanos_per_unit), column, builder)); + } + return Status::OK(); +} + +Status DataTypeTimeStampNsSerDe::read_column_from_arrow(IColumn& column, + const arrow::Array* arrow_array, + int64_t start, int64_t end, + const cctz::time_zone&) const { Review Comment: [P1] Honor timezone-bearing Arrow schemas Timezone-bearing Arrow schemas need a distinct conversion path here. TIMESTAMP_NS is timezone-naive, but this reader (and the writer above) ignores both `TimestampType::timezone()` and the supplied cctz zone, so `timestamp(ns, "+08:00")` value 0 is imported as the civil value `1970-01-01 00:00:00` instead of `1970-01-01 08:00:00` (and export shifts inversely). Please mirror DATETIMEV2's empty-timezone=UTC / nonempty-timezone=supplied-zone behavior at nanosecond precision, or explicitly reject timezone-bearing schemas, and cover a non-UTC negative-fraction case. ########## fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java: ########## @@ -712,9 +763,18 @@ private static List<Optional<DataType>> getInputImplicitCastTypes( List<Expression> inputs, List<DataType> expectedTypes) { Builder<Optional<DataType>> implicitCastTypes = ImmutableList.builderWithExpectedSize(inputs.size()); for (int i = 0; i < inputs.size(); i++) { - DataType argType = inputs.get(i).getDataType(); + Expression input = inputs.get(i); + DataType argType = input.getDataType(); DataType expectedType = expectedTypes.get(i); Optional<DataType> castType = TypeCoercionUtils.implicitCast(argType, expectedType); + // Signature resolution may select another date-like overload when a TIMESTAMP_NS + // literal is exactly representable by it. Keep argument coercion consistent with that + // value-aware decision without allowing TIMESTAMP_NS columns to lose precision. + if (!castType.isPresent() Review Comment: [P2] Apply exact nested timestamp casts This exact-literal fallback is scalar-only, but the new common-type search recurses into ARRAY/MAP/STRUCT. For example, `IF(cond, ARRAY(CAST('2024-01-02 03:04:05.123456000' AS TIMESTAMP_NS)), datetimev2_array_col)` now declares `ARRAY<DATETIMEV2(6)>`; recursive `implicitCast` rejects the TIMESTAMP_NS -> DATETIMEV2 leaf, and `argType` here is `ArrayType`, so the first branch remains `ARRAY<TIMESTAMP_NS>`. Final input-type checking therefore rejects an otherwise exact, supported expression. Please carry the value-aware cast decision through nested types and add analyzed/runtime tests with an exact TIMESTAMP_NS collection literal plus a nonliteral DATETIMEV2 collection (both branch orders). ########## fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnToProtobuf.java: ########## @@ -74,8 +74,15 @@ public static OlapFile.ColumnPB toPb(Column column, Set<String> bfColumns, List< builder.setAggregation("NONE"); } builder.setIsNullable(column.isAllowNull()); - if (column.getDefaultValue() != null) { - builder.setDefaultValue(ByteString.copyFrom(column.getDefaultValue().getBytes())); + String realDefaultValue = column.getRealDefaultValue(); Review Comment: [P1] Persist the Cloud on-update flag This Cloud schema path now preserves the TIMESTAMP_NS historical value and `default_value_expr`, but it still never sets `ColumnPB.is_on_update_current_timestamp`. `CloudInternalCatalog` persists this protobuf directly, so `TabletColumn` leaves the flag false; on `UPDATE_FLEXIBLE_COLUMNS`, both `FlexibleReadPlan` paths then keep the old cell instead of using the generated CURRENT_TIMESTAMP(7-9) value when the column is omitted. Please set `builder.setIsOnUpdateCurrentTimestamp(column.hasOnUpdateDefaultValue())` as `ColumnToThrift` does, and add a Cloud schema round-trip/flexible-update test for `DEFAULT CURRENT_TIMESTAMP(9) ON UPDATE CURRENT_TIMESTAMP(9)`. ########## be/test/core/data_type/data_type_timestamp_ns_test.cpp: ########## @@ -0,0 +1,512 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "core/data_type/data_type_timestamp_ns.h" + +#include <cctz/time_zone.h> +#include <gtest/gtest.h> + +#include <cstring> +#include <limits> +#include <memory> +#include <string> +#include <vector> + +#include "core/assert_cast.h" +#include "core/column/column_const.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/data_type/data_type_date_or_datetime_v2.h" +#include "core/data_type_serde/data_type_serde.h" +#include "core/data_type_serde/data_type_timestamp_ns_serde.h" +#include "core/string_buffer.hpp" +#include "core/value/vdatetime_value.h" +#include "exprs/function/cast/cast_parameters.h" +#include "storage/field_type.h" +#include "util/jsonb_utils.h" +#include "util/jsonb_writer.h" +#include "util/mysql_row_buffer.h" +#include "util/slice.h" +#include "util/timezone_utils.h" + +namespace doris { + +TEST(DataTypeTimeStampNsTest, TypeFamilyClassifiersKeepTimestampNsIndependent) { + EXPECT_FALSE(is_date_type(TYPE_TIMESTAMP_NS)); + EXPECT_TRUE(is_timestamp_ns_type(TYPE_TIMESTAMP_NS)); + EXPECT_FALSE(is_timestamp_ns_type(TYPE_DATETIMEV2)); + + EXPECT_TRUE(is_date_v2_or_datetime_v2(TYPE_DATEV2)); + EXPECT_TRUE(is_date_v2_or_datetime_v2(TYPE_DATETIMEV2)); + EXPECT_FALSE(is_date_v2_or_datetime_v2(TYPE_TIMESTAMP_NS)); + + EXPECT_TRUE(IsDataTypeDateTimeV2<DataTypeDateTimeV2>); + EXPECT_FALSE(IsDataTypeDateTimeV2<DataTypeTimeStampNs>); +} + +TEST(DataTypeTimeStampNsTest, Int64EpochRangeAndOrdering) { + const TimeStampNsValue epoch(0); + const TimeStampNsValue before_epoch(-1); + const TimeStampNsValue minimum(std::numeric_limits<int64_t>::min()); + const TimeStampNsValue maximum(std::numeric_limits<int64_t>::max()); + + EXPECT_EQ(epoch.to_string(), "1970-01-01 00:00:00.000000000"); + EXPECT_EQ(before_epoch.to_string(), "1969-12-31 23:59:59.999999999"); + EXPECT_EQ(minimum.to_string(), "1677-09-21 00:12:43.145224192"); + EXPECT_EQ(maximum.to_string(), "2262-04-11 23:47:16.854775807"); + EXPECT_LT(minimum, before_epoch); + EXPECT_LT(before_epoch, epoch); + EXPECT_LT(epoch, maximum); +} + +TEST(DataTypeTimeStampNsTest, NegativeEpochUsesFloorSecondAndNormalizedFraction) { + struct TestCase { + int64_t epoch_nanos; + int64_t epoch_seconds; + uint32_t nanosecond; + }; + const std::vector<TestCase> cases = { + {-1000000001, -2, 999999999}, + {-1000000000, -1, 0}, + {-999999999, -1, 1}, + {-1, -1, 999999999}, + {0, 0, 0}, + {1, 0, 1}, + }; + + for (const auto& test_case : cases) { + const TimeStampNsValue value(test_case.epoch_nanos); + EXPECT_EQ(value.epoch_seconds(), test_case.epoch_seconds); + EXPECT_EQ(value.nanosecond(), test_case.nanosecond); + EXPECT_EQ( + static_cast<__int128>(value.epoch_seconds()) * TimeStampNsValue::NANOS_PER_SECOND + + value.nanosecond(), + test_case.epoch_nanos); + } +} + +TEST(DataTypeTimeStampNsTest, ParseAtFixedNanosecondPrecision) { + int64_t value = 0; + + ASSERT_TRUE(parse_timestamp_ns(StringRef("1970-01-01 00:00:00.12345675"), &value).ok()); + EXPECT_EQ(TimeStampNsValue(value).to_string(), "1970-01-01 00:00:00.123456750"); + + ASSERT_TRUE(parse_timestamp_ns(StringRef("1969-12-31 23:59:59.999999999"), &value).ok()); + EXPECT_EQ(value, -1); + + ASSERT_TRUE(parse_timestamp_ns(StringRef("1970-01-01 00:00:00.999999995"), &value).ok()); + EXPECT_EQ(TimeStampNsValue(value).to_string(), "1970-01-01 00:00:00.999999995"); + + ASSERT_TRUE(parse_timestamp_ns(StringRef("1970-01-01 00:00:00.9999999995"), &value).ok()); + EXPECT_EQ(TimeStampNsValue(value).to_string(), "1970-01-01 00:00:01.000000000"); +} + +TEST(DataTypeTimeStampNsTest, ParseTimezoneSuffixInSessionTimezone) { + TimezoneUtils::load_timezones_to_cache(); + cctz::time_zone shanghai; + ASSERT_TRUE(cctz::load_time_zone("Asia/Shanghai", &shanghai)); + + int64_t value = 0; + auto status = + parse_timestamp_ns(StringRef("2023-08-17T01:41:18.123456789Z"), &value, &shanghai); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(TimeStampNsValue(value).to_string(), "2023-08-17 09:41:18.123456789"); + + ASSERT_TRUE(parse_timestamp_ns(StringRef("2023-08-17T01:41:18.123456789America/Los_Angeles"), + &value, &shanghai) + .ok()); + EXPECT_EQ(TimeStampNsValue(value).to_string(), "2023-08-17 16:41:18.123456789"); + + EXPECT_FALSE( + parse_timestamp_ns(StringRef("1677-09-21T00:12:43.145224192+14:00"), &value, &shanghai) + .ok()); + EXPECT_FALSE( + parse_timestamp_ns(StringRef("2262-04-11T23:47:16.854775807-01:00"), &value, &shanghai) + .ok()); +} + +TEST(DataTypeTimeStampNsTest, RoundZonedInputBeforeSessionTimezoneConversion) { + TimezoneUtils::load_timezones_to_cache(); + cctz::time_zone new_york; + ASSERT_TRUE(cctz::load_time_zone("America/New_York", &new_york)); + + int64_t value = 0; + ASSERT_TRUE(parse_timestamp_ns(StringRef("2024-03-10T06:59:59.9999999995Z"), &value, &new_york) + .ok()); + EXPECT_EQ(TimeStampNsValue(value).to_string(), "2024-03-10 03:00:00.000000000"); + + ASSERT_TRUE(parse_timestamp_ns(StringRef("2024-11-03T05:59:59.9999999995Z"), &value, &new_york) + .ok()); + EXPECT_EQ(TimeStampNsValue(value).to_string(), "2024-11-03 01:00:00.000000000"); +} + +TEST(DataTypeTimeStampNsTest, ParseAcceptsFractionalWidthsAndRejectsMalformedValues) { + struct ValidCase { + const char* input; + const char* expected; + }; + const std::vector<ValidCase> valid_cases = { + {"2024-02-29 12:34:56.1234567", "2024-02-29 12:34:56.123456700"}, + {"2024-02-29 12:34:56.12345678", "2024-02-29 12:34:56.123456780"}, + {"2024-02-29 12:34:56.123456789", "2024-02-29 12:34:56.123456789"}, + {"2024-02-29 12:34:56.1234567894", "2024-02-29 12:34:56.123456789"}, + {"2024-02-29 12:34:56.1234567895", "2024-02-29 12:34:56.123456790"}, + {"2024-02-29 12:34:56", "2024-02-29 12:34:56.000000000"}, + }; + + for (const auto& test_case : valid_cases) { + int64_t value = 0; + ASSERT_TRUE(parse_timestamp_ns(StringRef(test_case.input), &value).ok()) << test_case.input; + EXPECT_EQ(TimeStampNsValue(value).to_string(), test_case.expected); + } + + const std::vector<const char*> invalid_values = { + "", + "not-a-date", + "2023-02-29 00:00:00.000000000", + "2024-13-01", + "2024-01-01 24:00:00", + "2024-01-01 00:00:00.trailing", + "2024-01-01 00:00:00.123.456", + "2024-01-01.123 00:00:00", + "2024.01-01 00:00:00", + }; + for (const char* input : invalid_values) { + int64_t value = 0; + EXPECT_FALSE(parse_timestamp_ns(StringRef(input), &value).ok()) << input; + } +} + +TEST(DataTypeTimeStampNsTest, RejectValuesOutsideEpochRange) { + int64_t value = 0; + EXPECT_FALSE(parse_timestamp_ns(StringRef("0000-01-01 00:00:00.000000000"), &value).ok()); + EXPECT_FALSE(parse_timestamp_ns(StringRef("1677-09-21 00:12:43.145224191"), &value).ok()); + EXPECT_FALSE(parse_timestamp_ns(StringRef("2262-04-11 23:47:16.854775808"), &value).ok()); + EXPECT_FALSE(parse_timestamp_ns(StringRef("9999-12-31 23:59:59.999999999"), &value).ok()); +} + +TEST(DataTypeTimeStampNsTest, CivilRoundTripPreservesSubMicrosecondDigits) { + DateV2Value<DateTimeV2ValueType> civil; + civil.unchecked_set_time(2024, 2, 29, 23, 59, 58, 123456); + + TimeStampNsValue value; + ASSERT_TRUE(value.from_datetime(civil, 789)); + EXPECT_EQ(value.to_string(), "2024-02-29 23:59:58.123456789"); + const auto round_trip = value.to_datetime(); + EXPECT_EQ(round_trip.year(), 2024); + EXPECT_EQ(round_trip.month(), 2); + EXPECT_EQ(round_trip.day(), 29); + EXPECT_EQ(round_trip.hour(), 23); + EXPECT_EQ(round_trip.minute(), 59); + EXPECT_EQ(round_trip.second(), 58); + EXPECT_EQ(value.microsecond(), 123456); + EXPECT_EQ(value.nanosecond_remainder(), 789); + EXPECT_EQ(round_trip.to_date_int_val(), civil.to_date_int_val()); +} + +TEST(DataTypeTimeStampNsTest, CheckedIntervalAdditionRejectsEpochOverflow) { + const TimeStampNsValue minimum(std::numeric_limits<int64_t>::min()); + const TimeStampNsValue maximum(std::numeric_limits<int64_t>::max()); + const TimeInterval three_seconds(TimeUnit::SECOND, 3, false); + const TimeInterval negative_three_seconds(TimeUnit::SECOND, 3, true); + + auto below_minimum = minimum; + EXPECT_FALSE(below_minimum.date_add_interval<TimeUnit::SECOND>(negative_three_seconds)); + EXPECT_EQ(below_minimum, minimum); + + auto above_maximum = maximum; + EXPECT_FALSE(above_maximum.date_add_interval<TimeUnit::SECOND>(three_seconds)); + EXPECT_EQ(above_maximum, maximum); +} + +TEST(DataTypeTimeStampNsTest, FactoryKeepsTimestampNsSeparateFromDateTimeV2) { + const auto microseconds = create_datetimev2(6); + const auto timestamp_ns = std::make_shared<DataTypeTimeStampNs>(); + + EXPECT_EQ(microseconds->get_primitive_type(), TYPE_DATETIMEV2); + EXPECT_THROW(create_datetimev2(7), Exception); + EXPECT_THROW(create_datetimev2(8), Exception); + EXPECT_THROW(create_datetimev2(9), Exception); + EXPECT_EQ(timestamp_ns->get_primitive_type(), TYPE_TIMESTAMP_NS); + EXPECT_EQ(timestamp_ns->get_storage_field_type(), FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS); + EXPECT_EQ(timestamp_ns->get_scale(), 9); + EXPECT_EQ(microseconds->get_family_name(), "DateTimeV2"); + EXPECT_EQ(timestamp_ns->get_family_name(), "TimeStampNs"); +} + +TEST(DataTypeTimeStampNsTest, SerDeRoundTripsTextProtobufAndBinary) { + const auto type = std::make_shared<DataTypeTimeStampNs>(); + const auto serde = type->get_serde(); + auto source = type->create_column(); + DataTypeSerDe::FormatOptions options; + const std::vector<std::string> inputs = { + "1677-09-21 00:12:43.145224192", "1969-12-31 23:59:59.999999999", + "1970-01-01 00:00:00.000000000", "2024-02-29 12:34:56.123456789", + "2262-04-11 23:47:16.854775807", + }; + for (const auto& input : inputs) { + StringRef ref(input); + ASSERT_TRUE(serde->from_string(ref, *source, options).ok()) << input; + } + const auto& source_data = assert_cast<const ColumnTimeStampNs&>(*source).get_data(); + + PValues protobuf_values; + ASSERT_TRUE(serde->write_column_to_pb(*source, protobuf_values, 0, source->size()).ok()); + auto protobuf_result = type->create_column(); + ASSERT_TRUE(serde->read_column_from_pb(*protobuf_result, protobuf_values).ok()); + const auto& protobuf_data = assert_cast<const ColumnTimeStampNs&>(*protobuf_result).get_data(); + EXPECT_EQ(protobuf_data, source_data); + + ColumnString::Chars binary; + std::vector<size_t> offsets = {0}; + for (size_t row = 0; row < source->size(); ++row) { + serde->write_one_cell_to_binary(*source, binary, row); + offsets.push_back(binary.size()); + } + constexpr size_t bytes_per_row = sizeof(uint8_t) + sizeof(uint8_t) + sizeof(int64_t); + ASSERT_EQ(binary.size(), source->size() * bytes_per_row); + auto binary_result = ColumnNullable::create(type->create_column(), ColumnUInt8::create()); + for (size_t row = 0; row < source->size(); ++row) { + const uint8_t* begin = binary.data() + offsets[row]; + const uint8_t* end = DataTypeSerDe::deserialize_binary_to_column(begin, *binary_result); + EXPECT_EQ(end - begin, bytes_per_row); + } + const auto& binary_data = + assert_cast<const ColumnTimeStampNs&>(binary_result->get_nested_column()).get_data(); + EXPECT_EQ(binary_data, source_data); +} + +TEST(DataTypeTimeStampNsTest, SerDeExplicitlyRejectsUnsupportedArrowAndOrcIo) { + const DataTypeTimeStampNs type; + const auto serde = type.get_serde(); + auto column = type.create_column(); + const auto timezone = cctz::utc_time_zone(); + + const auto arrow_write_status = + serde->write_column_to_arrow(*column, nullptr, nullptr, 0, 0, timezone); Review Comment: [P1] Remove stale null Arrow calls This test now crashes instead of asserting a rejection. The same PR implements TIMESTAMP_NS Arrow I/O: the writer immediately dereferences `array_builder` (and calls `array_builder->type()`), and the reader dereferences `arrow_array`, but both calls here pass `nullptr`. Please remove these stale Arrow-rejection checks or replace them with valid Arrow objects/round-trip assertions; the ORC rejection can remain. ########## fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicate.java: ########## @@ -321,17 +353,43 @@ private static Expression processDateTimeLikeComparisonPredicateDateLiteral( return comparisonPredicate; } + /** + * Simplify a date/datetime comparison whose right literal is outside the range representable by + * the left expression. For every non-null left value, the result is constant and depends only on + * the comparison direction and whether the literal is below the minimum or above the maximum. + * Null-safe equality is always false; other predicates return true-or-null or false-or-null to + * preserve SQL three-valued logic when the left expression is nullable. + * + * @param comparisonPredicate comparison to simplify + * @param left left expression whose type defines the representable range + * @param rightBeforeMin true if the right literal is below that range; false if it is above + * @return the constant-equivalent expression with the original NULL semantics + */ + private static Expression simplifyDateComparisonOutsideRange( + ComparisonPredicate comparisonPredicate, Expression left, boolean rightBeforeMin) { + if (comparisonPredicate instanceof NullSafeEqual) { + return BooleanLiteral.FALSE; + } + if (comparisonPredicate instanceof EqualTo) { + return ExpressionUtils.falseOrNull(left); + } + boolean alwaysTrue = rightBeforeMin + ? comparisonPredicate instanceof GreaterThan || comparisonPredicate instanceof GreaterThanEqual + : comparisonPredicate instanceof LessThan || comparisonPredicate instanceof LessThanEqual; + return alwaysTrue ? ExpressionUtils.trueOrNull(left) : ExpressionUtils.falseOrNull(left); + } + // process cast(date as datetime/date) cmp datetime/date private static Expression processDateLikeComparisonPredicateDateLiteral( ComparisonPredicate comparisonPredicate, Expression left, DateLiteral right) { if (!(left.getDataType() instanceof DateType) && !(left.getDataType() instanceof DateV2Type)) { return comparisonPredicate; } - if (right instanceof DateTimeLiteral) { - DateTimeLiteral dateTimeLiteral = (DateTimeLiteral) right; - right = migrateToDateV2(dateTimeLiteral); - if (dateTimeLiteral.getHour() != 0 || dateTimeLiteral.getMinute() != 0 - || dateTimeLiteral.getSecond() != 0 || dateTimeLiteral.getMicroSecond() != 0) { + if (right.getDataType().isDateTimeType() || right.getDataType().isDateTimeV2Type() + || right.getDataType().isTimeStampNsType() || right.getDataType().isTimeStampTzType()) { Review Comment: [P1] Preserve partial DATE-to-TIMESTAMP_NS casts This also strips DATE/DATEV2 -> TIMESTAMP_NS casts, but those casts are partial because TIMESTAMP_NS only covers 1677-2262. For example, `CAST(date_value AS TIMESTAMP_NS) > TIMESTAMP_NS '2262-04-11 23:47:16.854775807'` is rewritten to `date_value > DATEV2 '2262-04-11'`; `9999-12-31` therefore becomes TRUE instead of NULL in non-strict mode or a TIMESTAMP_NS overflow in strict mode. Please retain this cast unless the source is proven within range, matching the guard already used for DATETIME/DATETIMEV2, and add out-of-range filter/project tests. ########## fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java: ########## @@ -1444,6 +1537,112 @@ public static Expression processInPredicate(InPredicate inPredicate) { .orElse(fmtInPredicate); } + private static Optional<Expression> normalizeTimeStampNsDateLikeComparison( + ComparisonPredicate comparisonPredicate) { + Expression left = comparisonPredicate.left(); + Expression right = comparisonPredicate.right(); + if (left.getDataType() instanceof TimeStampNsType + && canWidenLosslesslyToDateTimeV2(right.getDataType())) { + return Optional.of(comparisonPredicate.withChildren( + left, castIfNotSameType(right, DateTimeV2Type.forType(right.getDataType())))); + } + if (right.getDataType() instanceof TimeStampNsType + && canWidenLosslesslyToDateTimeV2(left.getDataType())) { + return Optional.of(comparisonPredicate.withChildren( + castIfNotSameType(left, DateTimeV2Type.forType(left.getDataType())), right)); + } + return Optional.empty(); + } + + private static boolean canWidenLosslesslyToDateTimeV2(DataType dataType) { + return dataType instanceof DateTimeV2Type + || dataType instanceof DateTimeType + || dataType instanceof DateV2Type + || dataType instanceof DateType; + } + + private static DateTimeV2Type widestDateTimeV2Type(List<Expression> expressions) { + int scale = 0; + for (Expression expression : expressions) { + if (expression.getDataType() instanceof DateTimeV2Type) { + scale = Math.max(scale, ((DateTimeV2Type) expression.getDataType()).getScale()); + } + } + return DateTimeV2Type.of(scale); + } + + /** + * Normalize mixed TIMESTAMP_NS/date-like options independently when the whole IN list has no + * common type. DATE, DATEV2, and DATETIME operands are widened losslessly to DATETIMEV2. + * Exactly representable literals are cast to the compare expression's type, impossible literals + * are removed, and non-literal mixed options are lowered to exact mixed equalities. Keeping NULL + * in the homogeneous IN portion preserves IN and NOT IN three-valued semantics. + */ + private static Optional<Expression> normalizeTimeStampNsDateLikeInOptions(InPredicate inPredicate) { + Expression originalCompareExpr = inPredicate.getCompareExpr(); + DataType originalCompareType = originalCompareExpr.getDataType(); + if (!(originalCompareType instanceof TimeStampNsType) + && !canWidenLosslesslyToDateTimeV2(originalCompareType)) { + return Optional.empty(); + } + + DateTimeV2Type dateTimeV2Type = widestDateTimeV2Type(inPredicate.children()); + Expression compareExpr = originalCompareType instanceof TimeStampNsType + ? originalCompareExpr : castIfNotSameType(originalCompareExpr, dateTimeV2Type); + DataType compareType = compareExpr.getDataType(); + List<Expression> normalizedOptions = new ArrayList<>(inPredicate.getOptions().size()); + List<Expression> mixedEqualities = new ArrayList<>(); + for (Expression option : inPredicate.getOptions()) { + if (option.isNullLiteral() || option.getDataType().equals(compareType)) { + normalizedOptions.add(castIfNotSameType(option, compareType)); + continue; + } + Expression normalizedOption = option; + if (canWidenLosslesslyToDateTimeV2(option.getDataType())) { + normalizedOption = castIfNotSameType(option, dateTimeV2Type); + } + if (normalizedOption.getDataType().equals(compareType)) { + normalizedOptions.add(normalizedOption); + continue; + } + if (!isTimeStampNsAndDateTimeV2Pair(compareType, normalizedOption.getDataType())) { + return Optional.empty(); + } + // Only a successfully evaluated literal proves that a failed exact conversion means + // the equality is impossible. Non-literals use the exact mixed comparison kernel. + Optional<Literal> optionLiteral = getLiteralAfterExplicitCast(option); + if (!optionLiteral.isPresent()) { + mixedEqualities.add(processComparisonPredicate(new EqualTo(compareExpr, normalizedOption))); + continue; + } + Literal literal = optionLiteral.get(); + if (literal.isNullLiteral()) { + normalizedOptions.add(castIfNotSameType(option, compareType)); + continue; + } + boolean exactlyRepresentable = compareType instanceof TimeStampNsType + ? canExactlyCastLiteralToTimeStampNs(literal) + : literal instanceof TimeStampNsLiteral + && canExactlyCastTimeStampNsLiteralTo( + (TimeStampNsLiteral) literal, compareType); + if (exactlyRepresentable) { + normalizedOptions.add(castIfNotSameType(option, compareType)); + } + } + List<Expression> disjunctions = new ArrayList<>(); + if (!normalizedOptions.isEmpty()) { + disjunctions.add(new InPredicate(compareExpr, normalizedOptions)); + } + disjunctions.addAll(mixedEqualities); + if (disjunctions.isEmpty()) { + return Optional.of(ExpressionUtils.falseOrNull(compareExpr)); + } + if (disjunctions.size() > 1 && compareExpr.containsVolatileExpression()) { Review Comment: [P2] Allow volatile compare values in mixed IN lowering This guard makes a legal predicate such as `CAST(random(19700101, 19700102) AS TIMESTAMP_NS) IN (datetimev2_col, TIMESTAMP_NS '1970-01-01 00:00:00.000000000')` fail analysis. Normalization has already built a same-type IN plus an exact mixed equality, but returning empty here makes `processInPredicate` throw `unsupported in predicate`. Both branches reuse the same volatile identity, and `AddProjectForVolatileExpression` later materializes repeated volatile functions once, so single evaluation is already supported. Please let that rule handle this lowering (or materialize explicitly here), and add IN/NOT IN coverage including NULL. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
