This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/doris.git
commit a8c24d7698f89a1f9ebb63e6e01ff78bbc73a575 Author: zclllyybb <[email protected]> AuthorDate: Tue May 21 20:27:09 2024 +0800 [Fix](function) fix overflow of date_add function (#35080) fix overflow of date_add function --- be/src/common/status.h | 7 ++ be/src/vec/runtime/vdatetime_value.cpp | 74 ++++++++++------------ be/src/vec/runtime/vdatetime_value.h | 43 +++++++++++-- .../correctness/test_date_function_const.groovy | 5 ++ .../datetime_functions/test_date_function.groovy | 11 ++-- 5 files changed, 87 insertions(+), 53 deletions(-) diff --git a/be/src/common/status.h b/be/src/common/status.h index cf9b42a3c69..6e5a75f7966 100644 --- a/be/src/common/status.h +++ b/be/src/common/status.h @@ -619,6 +619,13 @@ inline std::string Status::to_string_no_stack() const { } \ } while (false) +#define PROPAGATE_FALSE(stmt) \ + do { \ + if (UNLIKELY(!static_cast<bool>(stmt))) { \ + return false; \ + } \ + } while (false) + #define THROW_IF_ERROR(stmt) \ do { \ Status _status_ = (stmt); \ diff --git a/be/src/vec/runtime/vdatetime_value.cpp b/be/src/vec/runtime/vdatetime_value.cpp index fcdaedea183..dd0ce341493 100644 --- a/be/src/vec/runtime/vdatetime_value.cpp +++ b/be/src/vec/runtime/vdatetime_value.cpp @@ -42,8 +42,6 @@ namespace doris { -static constexpr int s_days_in_month[13] = {0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}; - static const char* s_ab_month_name[] = {"", "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec", nullptr}; @@ -74,7 +72,7 @@ bool VecDateTimeValue::check_range(uint32_t year, uint32_t month, uint32_t day, bool VecDateTimeValue::check_date(uint32_t year, uint32_t month, uint32_t day) { if (month == 2 && day == 29 && doris::is_leap(year)) return false; - if (year > 9999 || month == 0 || month > 12 || day > s_days_in_month[month] || day == 0) { + if (year > 9999 || month == 0 || month > 12 || day > S_DAYS_IN_MONTH[month] || day == 0) { return true; } return false; @@ -520,8 +518,8 @@ bool VecDateTimeValue::get_date_from_daynr(uint64_t daynr) { } } month = 1; - while (days_of_year > s_days_in_month[month]) { - days_of_year -= s_days_in_month[month]; + while (days_of_year > S_DAYS_IN_MONTH[month]) { + days_of_year -= S_DAYS_IN_MONTH[month]; month++; } day = days_of_year + leap_day; @@ -1679,8 +1677,8 @@ bool VecDateTimeValue::date_add_interval(const TimeInterval& interval) { return false; } _month = (months % 12) + 1; - if (_day > s_days_in_month[_month]) { - _day = s_days_in_month[_month]; + if (_day > S_DAYS_IN_MONTH[_month]) { + _day = S_DAYS_IN_MONTH[_month]; if (_month == 2 && doris::is_leap(_year)) { _day++; } @@ -1931,11 +1929,11 @@ bool DateV2Value<T>::is_invalid(uint32_t year, uint32_t month, uint32_t day, uin if (only_time_part) { return false; } - if (year < MIN_YEAR || year > MAX_YEAR) { + if (year > MAX_YEAR) { return true; } if (month == 2 && day == 29 && doris::is_leap(year)) return false; - if (month == 0 || month > 12 || day > s_days_in_month[month] || day == 0) { + if (month == 0 || month > 12 || day > S_DAYS_IN_MONTH[month] || day == 0) { return true; } return false; @@ -1960,7 +1958,7 @@ void DateV2Value<T>::format_datetime(uint32_t* date_val, bool* carry_bits) const date_val[1] += 1; carry_bits[2] = true; } - } else if (date_val[2] == s_days_in_month[date_val[1]] + 1 && carry_bits[3]) { + } else if (date_val[2] == S_DAYS_IN_MONTH[date_val[1]] + 1 && carry_bits[3]) { date_val[2] = 1; date_val[1] += 1; carry_bits[2] = true; @@ -2916,8 +2914,8 @@ bool DateV2Value<T>::get_date_from_daynr(uint64_t daynr) { } } month = 1; - while (days_of_year > s_days_in_month[month]) { - days_of_year -= s_days_in_month[month]; + while (days_of_year > S_DAYS_IN_MONTH[month]) { + days_of_year -= S_DAYS_IN_MONTH[month]; month++; } day = days_of_year + leap_day; @@ -2974,30 +2972,27 @@ bool DateV2Value<T>::date_add_interval(const TimeInterval& interval, DateV2Value to_value.set_time(seconds / 3600, (seconds / 60) % 60, seconds % 60, microseconds); } else if constexpr (unit == YEAR) { // This only change year information - to_value.template set_time_unit<TimeUnit::YEAR>(date_v2_value_.year_ + interval.year); - if (to_value.year() > 9999) { - return false; - } + PROPAGATE_FALSE(to_value.template set_time_unit<TimeUnit::YEAR>(date_v2_value_.year_ + + interval.year)); if (date_v2_value_.month_ == 2 && date_v2_value_.day_ == 29 && !doris::is_leap(to_value.year())) { - to_value.template set_time_unit<TimeUnit::DAY>(28); + // add year. so if from Leap Year to Equal Year, use last day of Feb(29 to 28) + PROPAGATE_FALSE(to_value.template set_time_unit<TimeUnit::DAY>(28)); } } else if constexpr (unit == QUARTER || unit == MONTH || unit == YEAR_MONTH) { // This will change month and year information, maybe date. int64_t months = date_v2_value_.year_ * 12 + date_v2_value_.month_ - 1 + 12 * interval.year + interval.month; - to_value.template set_time_unit<TimeUnit::YEAR>(months / 12); if (months < 0) { return false; } - if (to_value.year() > MAX_YEAR) { - return false; - } - to_value.template set_time_unit<TimeUnit::MONTH>((months % 12) + 1); - if (date_v2_value_.day_ > s_days_in_month[to_value.month()]) { - date_v2_value_.day_ = s_days_in_month[to_value.month()]; + PROPAGATE_FALSE(to_value.template set_time_unit<TimeUnit::YEAR>(months / 12)); + PROPAGATE_FALSE(to_value.template set_time_unit<TimeUnit::MONTH>((months % 12) + 1)); + if (date_v2_value_.day_ > S_DAYS_IN_MONTH[to_value.month()]) { + date_v2_value_.day_ = S_DAYS_IN_MONTH[to_value.month()]; if (to_value.month() == 2 && doris::is_leap(to_value.year())) { - to_value.template set_time_unit<TimeUnit::DAY>(date_v2_value_.day_ + 1); + PROPAGATE_FALSE( + to_value.template set_time_unit<TimeUnit::DAY>(date_v2_value_.day_ + 1)); } } } @@ -3050,30 +3045,27 @@ bool DateV2Value<T>::date_add_interval(const TimeInterval& interval) { } } else if constexpr (unit == YEAR) { // This only change year information - this->template set_time_unit<TimeUnit::YEAR>(date_v2_value_.year_ + interval.year); - if (this->year() > 9999) { - return false; - } + PROPAGATE_FALSE( + this->template set_time_unit<TimeUnit::YEAR>(date_v2_value_.year_ + interval.year)); if (date_v2_value_.month_ == 2 && date_v2_value_.day_ == 29 && !doris::is_leap(this->year())) { - this->template set_time_unit<TimeUnit::DAY>(28); + // add year. so if from Leap Year to Equal Year, use last day of Feb(29 to 28) + PROPAGATE_FALSE(this->template set_time_unit<TimeUnit::DAY>(28)); } } else if constexpr (unit == QUARTER || unit == MONTH || unit == YEAR_MONTH) { // This will change month and year information, maybe date. int64_t months = date_v2_value_.year_ * 12 + date_v2_value_.month_ - 1 + 12 * interval.year + interval.month; - this->template set_time_unit<TimeUnit::YEAR>(months / 12); if (months < 0) { return false; } - if (this->year() > MAX_YEAR) { - return false; - } - this->template set_time_unit<TimeUnit::MONTH>((months % 12) + 1); - if (date_v2_value_.day_ > s_days_in_month[this->month()]) { - date_v2_value_.day_ = s_days_in_month[this->month()]; + PROPAGATE_FALSE(this->template set_time_unit<TimeUnit::YEAR>(months / 12)); + PROPAGATE_FALSE(this->template set_time_unit<TimeUnit::MONTH>((months % 12) + 1)); + if (date_v2_value_.day_ > S_DAYS_IN_MONTH[this->month()]) { + date_v2_value_.day_ = S_DAYS_IN_MONTH[this->month()]; if (this->month() == 2 && doris::is_leap(this->year())) { - this->template set_time_unit<TimeUnit::DAY>(date_v2_value_.day_ + 1); + PROPAGATE_FALSE( + this->template set_time_unit<TimeUnit::DAY>(date_v2_value_.day_ + 1)); } } } @@ -3102,13 +3094,13 @@ bool DateV2Value<T>::date_set_interval(const TimeInterval& interval) { } } else if constexpr (unit == YEAR) { this->set_time(0, 1, 1, 0, 0, 0, 0); - this->template set_time_unit<TimeUnit::YEAR>(interval.year); + PROPAGATE_FALSE(this->template set_time_unit<TimeUnit::YEAR>(interval.year)); } else if constexpr (unit == MONTH) { // This will change month and year information, maybe date. this->set_time(0, 1, 1, 0, 0, 0, 0); int64_t months = 12 * interval.year + interval.month; - this->template set_time_unit<TimeUnit::YEAR>(months / 12); - this->template set_time_unit<TimeUnit::MONTH>((months % 12) + 1); + PROPAGATE_FALSE(this->template set_time_unit<TimeUnit::YEAR>(months / 12)); + PROPAGATE_FALSE(this->template set_time_unit<TimeUnit::MONTH>((months % 12) + 1)); } return true; } diff --git a/be/src/vec/runtime/vdatetime_value.h b/be/src/vec/runtime/vdatetime_value.h index b1197b611ed..8fcb45a6440 100644 --- a/be/src/vec/runtime/vdatetime_value.h +++ b/be/src/vec/runtime/vdatetime_value.h @@ -19,15 +19,13 @@ #include <glog/logging.h> #include <re2/re2.h> -#include <stdint.h> -#include <string.h> #include <algorithm> #include <cstddef> #include <cstdint> +#include <cstring> #include <iostream> #include <iterator> -#include <shared_mutex> #include <string> #include <string_view> #include <tuple> @@ -168,6 +166,8 @@ constexpr int HOUR_PER_DAY = 24; constexpr int64_t SECOND_PER_HOUR = 3600; constexpr int64_t SECOND_PER_MINUTE = 60; +inline constexpr int S_DAYS_IN_MONTH[13] = {0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}; + constexpr size_t const_length(const char* str) { return (str == nullptr || *str == 0) ? 0 : const_length(str + 1) + 1; } @@ -201,7 +201,11 @@ static constexpr uint64_t MAX_DATETIME_V2 = ((uint64_t)MAX_DATE_V2 << TIME_PART_ static constexpr uint64_t MIN_DATETIME_V2 = (uint64_t)MIN_DATE_V2 << TIME_PART_LENGTH; static constexpr uint32_t MAX_YEAR = 9999; -static constexpr uint32_t MIN_YEAR = 0; +static constexpr uint32_t MAX_MONTH = 12; +static constexpr uint32_t MAX_HOUR = 23; +static constexpr uint32_t MAX_MINUTE = 59; +static constexpr uint32_t MAX_SECOND = 59; +static constexpr uint32_t MAX_MICROSECOND = 999999; static constexpr uint32_t DATEV2_YEAR_WIDTH = 23; static constexpr uint32_t DATETIMEV2_YEAR_WIDTH = 18; @@ -1159,31 +1163,58 @@ public: bool get_date_from_daynr(uint64_t); + // should do check template <TimeUnit unit> - void set_time_unit(uint32_t val) { + bool set_time_unit(uint32_t val) { + // is uint so need check upper bound only if constexpr (unit == TimeUnit::YEAR) { + if (val > MAX_YEAR) [[unlikely]] { + return false; + } date_v2_value_.year_ = val; } else if constexpr (unit == TimeUnit::MONTH) { + if (val > MAX_MONTH) [[unlikely]] { + return false; + } date_v2_value_.month_ = val; } else if constexpr (unit == TimeUnit::DAY) { + DCHECK(date_v2_value_.month_ <= MAX_MONTH); + DCHECK(date_v2_value_.month_ != 0); + if (val > S_DAYS_IN_MONTH[date_v2_value_.month_] && + !(is_leap(date_v2_value_.year_) && date_v2_value_.month_ == 2 && val == 29)) { + return false; + } date_v2_value_.day_ = val; } else if constexpr (unit == TimeUnit::HOUR) { if constexpr (is_datetime) { + if (val > MAX_HOUR) [[unlikely]] { + return false; + } date_v2_value_.hour_ = val; } } else if constexpr (unit == TimeUnit::MINUTE) { if constexpr (is_datetime) { + if (val > MAX_MINUTE) [[unlikely]] { + return false; + } date_v2_value_.minute_ = val; } } else if constexpr (unit == TimeUnit::SECOND) { if constexpr (is_datetime) { + if (val > MAX_SECOND) [[unlikely]] { + return false; + } date_v2_value_.second_ = val; } - } else if constexpr (unit == TimeUnit::SECOND_MICROSECOND) { + } else if constexpr (unit == TimeUnit::MICROSECOND) { if constexpr (is_datetime) { + if (val > MAX_MICROSECOND) [[unlikely]] { + return false; + } date_v2_value_.microsecond_ = val; } } + return true; } operator int64_t() const { return to_int64(); } diff --git a/regression-test/suites/correctness/test_date_function_const.groovy b/regression-test/suites/correctness/test_date_function_const.groovy index 0caded34f5d..d1ba4db4e68 100644 --- a/regression-test/suites/correctness/test_date_function_const.groovy +++ b/regression-test/suites/correctness/test_date_function_const.groovy @@ -58,4 +58,9 @@ suite("test_date_function_const") { sql("""select date_add(CURRENT_DATE(),-2);""") notContains("00:00:00") } + + test { + sql """select date_add("1900-01-01 12:00:00.123456", interval 10000000000 month);""" + exception "Operation months_add 133705200962757184 1410065408 out of range" + } } diff --git a/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function.groovy b/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function.groovy index d1e8fcab805..2f1ef98b4ea 100644 --- a/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function.groovy +++ b/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function.groovy @@ -600,9 +600,9 @@ suite("test_date_function") { qt_sql """ select seconds_sub(test_time2,1) result from ${tableName}; """ // test last_day for vec - sql """ DROP TABLE IF EXISTS ${tableName}; """ + sql """ DROP TABLE IF EXISTS test_time_add_sub_function; """ sql """ - CREATE TABLE IF NOT EXISTS ${tableName} ( + CREATE TABLE IF NOT EXISTS test_time_add_sub_function ( birth date, birth1 datev2, birth2 datetime, @@ -612,7 +612,7 @@ suite("test_date_function") { PROPERTIES( "replication_allocation" = "tag.location.default: 1"); """ sql """ - insert into ${tableName} values + insert into test_time_add_sub_function values ('2022-01-01', '2022-01-01', '2022-01-01 00:00:00', '2022-01-01 00:00:00'), ('2000-02-01', '2000-02-01', '2000-02-01 00:00:00', '2000-02-01 00:00:00.123'), ('2022-02-27', '2022-02-27', '2022-02-27 00:00:00', '2022-02-27 00:00:00'), @@ -620,11 +620,10 @@ suite("test_date_function") { qt_sql """ select last_day(birth), last_day(birth1), last_day(birth2), last_day(birth3) - from ${tableName}; + from test_time_add_sub_function; """ - sql """ DROP TABLE IF EXISTS ${tableName}; """ - sql """ DROP TABLE IF EXISTS ${tableName}; """ + sql """ DROP TABLE IF EXISTS test_time_add_sub_function; """ sql """ CREATE TABLE IF NOT EXISTS ${tableName} ( birth date, --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
