This is an automated email from the ASF dual-hosted git repository.
gabriellee pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 8973610543 [feature](datetime) "timediff" supports calculating
microseconds (#21371)
8973610543 is described below
commit 89736105434c23787afd412ce808a17ca55d6cf0
Author: Mryange <[email protected]>
AuthorDate: Mon Jul 10 19:21:32 2023 +0800
[feature](datetime) "timediff" supports calculating microseconds (#21371)
---
be/src/runtime/types.cpp | 2 +-
be/src/util/date_func.cpp | 94 +++++++++++++++++++++
be/src/util/date_func.h | 3 +-
be/src/util/mysql_row_buffer.cpp | 26 ++++++
be/src/util/mysql_row_buffer.h | 1 +
be/src/vec/core/call_on_type_index.h | 2 +
be/src/vec/data_types/data_type.cpp | 2 +
be/src/vec/data_types/data_type_factory.cpp | 10 ++-
be/src/vec/data_types/data_type_time.cpp | 28 +++++++
be/src/vec/data_types/data_type_time.h | 36 ++++++++
.../vec/data_types/serde/data_type_time_serde.cpp | 21 +++++
be/src/vec/data_types/serde/data_type_time_serde.h | 14 ++++
be/src/vec/exec/format/parquet/schema_desc.cpp | 2 +-
.../function_date_or_datetime_computation.h | 43 +++++++++-
be/src/vec/functions/function_running_difference.h | 4 +-
be/src/vec/runtime/vdatetime_value.h | 20 +++++
be/src/vec/sink/vmysql_result_writer.cpp | 5 +-
be/test/vec/function/function_time_test.cpp | 7 +-
.../java/org/apache/doris/catalog/ScalarType.java | 13 ++-
.../main/java/org/apache/doris/analysis/Expr.java | 4 +-
.../apache/doris/analysis/FunctionCallExpr.java | 7 ++
.../expressions/functions/scalar/TimeDiff.java | 42 +++++++++-
.../org/apache/doris/nereids/types/TimeV2Type.java | 20 ++++-
gensrc/proto/types.proto | 1 +
.../correctness/test_time_diff_microseconds.out | 29 +++++++
.../data/nereids_p0/test_char_implicit_cast.out | 6 +-
.../correctness/test_time_diff_microseconds.groovy | 95 ++++++++++++++++++++++
.../nereids_p0/test_char_implicit_cast.groovy | 4 +-
28 files changed, 519 insertions(+), 22 deletions(-)
diff --git a/be/src/runtime/types.cpp b/be/src/runtime/types.cpp
index ebcb6eb378..8cbeebe4c3 100644
--- a/be/src/runtime/types.cpp
+++ b/be/src/runtime/types.cpp
@@ -46,7 +46,7 @@ TypeDescriptor::TypeDescriptor(const std::vector<TTypeNode>&
types, int* idx)
DCHECK(scalar_type.__isset.len);
len = scalar_type.len;
} else if (type == TYPE_DECIMALV2 || type == TYPE_DECIMAL32 || type ==
TYPE_DECIMAL64 ||
- type == TYPE_DECIMAL128I || type == TYPE_DATETIMEV2) {
+ type == TYPE_DECIMAL128I || type == TYPE_DATETIMEV2 || type
== TYPE_TIMEV2) {
DCHECK(scalar_type.__isset.precision);
DCHECK(scalar_type.__isset.scale);
precision = scalar_type.precision;
diff --git a/be/src/util/date_func.cpp b/be/src/util/date_func.cpp
index e7ae30e003..6ca199d630 100644
--- a/be/src/util/date_func.cpp
+++ b/be/src/util/date_func.cpp
@@ -86,6 +86,7 @@ uint64_t timestamp_from_datetime_v2(const std::string&
date_str) {
}
// refer to https://dev.mysql.com/doc/refman/5.7/en/time.html
// the time value between '-838:59:59' and '838:59:59'
+/// TODO: Why is the time type stored as double? Can we directly use int64 and
remove the time limit?
int32_t time_to_buffer_from_double(double time, char* buffer) {
char* begin = buffer;
if (time < 0) {
@@ -115,6 +116,52 @@ int32_t time_to_buffer_from_double(double time, char*
buffer) {
return buffer - begin;
}
+int32_t timev2_to_buffer_from_double(double time, char* buffer, int scale) {
+ static int pow10[7] = {1, 10, 100, 1000, 10000, 100000, 1000000};
+ char* begin = buffer;
+ if (time < 0) {
+ time = -time;
+ *buffer++ = '-';
+ }
+ int64_t m_time = time;
+ // m_time = hour * 3600 * 1000 * 1000 + minute * 60 * 1000 * 1000 + second
* 1000 * 1000 + microsecond
+ int64_t hour = m_time / ((int64_t)3600 * 1000 * 1000);
+ if (hour >= 100) {
+ auto f = fmt::format_int(hour);
+ memcpy(buffer, f.data(), f.size());
+ buffer = buffer + f.size();
+ } else {
+ *buffer++ = (char)('0' + (hour / 10));
+ *buffer++ = (char)('0' + (hour % 10));
+ }
+ *buffer++ = ':';
+ m_time %= (int64_t)3600 * 1000 * 1000;
+ int64_t minute = m_time / (60 * 1000 * 1000);
+ *buffer++ = (char)('0' + (minute / 10));
+ *buffer++ = (char)('0' + (minute % 10));
+ *buffer++ = ':';
+ m_time %= 60 * 1000 * 1000;
+ int32_t second = m_time / (1000 * 1000);
+ *buffer++ = (char)('0' + (second / 10));
+ *buffer++ = (char)('0' + (second % 10));
+ m_time %= 1000 * 1000;
+ if (scale == 0) {
+ return buffer - begin;
+ }
+ *buffer++ = '.';
+ memset(buffer, '0', scale);
+ buffer += scale;
+ int32_t micosecond = m_time % (1000 * 1000);
+ micosecond /= pow10[6 - scale];
+ auto it = buffer - 1;
+ while (micosecond) {
+ *it = (char)('0' + (micosecond % 10));
+ micosecond /= 10;
+ it--;
+ }
+ return buffer - begin;
+}
+
std::string time_to_buffer_from_double(double time) {
fmt::memory_buffer buffer;
if (time < 0) {
@@ -136,4 +183,51 @@ std::string time_to_buffer_from_double(double time) {
return fmt::to_string(buffer);
}
+std::string timev2_to_buffer_from_double(double time, int scale) {
+ static int pow10[7] = {1, 10, 100, 1000, 10000, 100000, 1000000};
+ fmt::memory_buffer buffer;
+ if (time < 0) {
+ time = -time;
+ fmt::format_to(buffer, "-");
+ }
+ int64_t m_time = time;
+ // m_time = hour * 3600 * 1000 * 1000 + minute * 60 * 1000 * 1000 + second
* 1000 * 1000 + microsecond
+ int64_t hour = m_time / ((int64_t)3600 * 1000 * 1000);
+ if (hour >= 100) {
+ fmt::format_to(buffer, fmt::format("{}", hour));
+ } else {
+ fmt::format_to(buffer, fmt::format("{:02d}", hour));
+ }
+ m_time %= (int64_t)3600 * 1000 * 1000;
+ int64_t minute = m_time / (60 * 1000 * 1000);
+ m_time %= 60 * 1000 * 1000;
+ int32_t second = m_time / (1000 * 1000);
+ int32_t micosecond = m_time % (1000 * 1000);
+ micosecond /= pow10[6 - scale];
+ switch (scale) {
+ case 0:
+ fmt::format_to(buffer, fmt::format(":{:02d}:{:02d}", minute, second,
micosecond));
+ break;
+ case 1:
+ fmt::format_to(buffer, fmt::format(":{:02d}:{:02d}.{:01d}", minute,
second, micosecond));
+ break;
+ case 2:
+ fmt::format_to(buffer, fmt::format(":{:02d}:{:02d}.{:02d}", minute,
second, micosecond));
+ break;
+ case 3:
+ fmt::format_to(buffer, fmt::format(":{:02d}:{:02d}.{:03d}", minute,
second, micosecond));
+ break;
+ case 4:
+ fmt::format_to(buffer, fmt::format(":{:02d}:{:02d}.{:04d}", minute,
second, micosecond));
+ break;
+ case 5:
+ fmt::format_to(buffer, fmt::format(":{:02d}:{:02d}.{:05d}", minute,
second, micosecond));
+ break;
+ case 6:
+ fmt::format_to(buffer, fmt::format(":{:02d}:{:02d}.{:06d}", minute,
second, micosecond));
+ break;
+ }
+
+ return fmt::to_string(buffer);
+}
} // namespace doris
diff --git a/be/src/util/date_func.h b/be/src/util/date_func.h
index cc9aa7b6da..405132c3f0 100644
--- a/be/src/util/date_func.h
+++ b/be/src/util/date_func.h
@@ -26,9 +26,10 @@ namespace doris {
uint64_t timestamp_from_datetime(const std::string& datetime_str);
uint32_t timestamp_from_date(const std::string& date_str);
int32_t time_to_buffer_from_double(double time, char* buffer);
+int32_t timev2_to_buffer_from_double(double time, char* buffer, int scale);
uint32_t timestamp_from_date_v2(const std::string& date_str);
uint64_t timestamp_from_datetime_v2(const std::string& date_str);
std::string time_to_buffer_from_double(double time);
-
+std::string timev2_to_buffer_from_double(double time, int scale);
} // namespace doris
diff --git a/be/src/util/mysql_row_buffer.cpp b/be/src/util/mysql_row_buffer.cpp
index a8d2674073..00193481b7 100644
--- a/be/src/util/mysql_row_buffer.cpp
+++ b/be/src/util/mysql_row_buffer.cpp
@@ -195,6 +195,13 @@ static char* add_time(double data, char* pos, bool
dynamic_mode) {
return pos + length;
}
+static char* add_timev2(double data, char* pos, bool dynamic_mode, int scale) {
+ int length = timev2_to_buffer_from_double(data, pos + !dynamic_mode,
scale);
+ if (!dynamic_mode) {
+ int1store(pos++, length);
+ }
+ return pos + length;
+}
template <typename DateType>
static char* add_datetime(const DateType& data, char* pos, bool dynamic_mode) {
int length = data.to_buffer(pos + !dynamic_mode);
@@ -422,6 +429,25 @@ int MysqlRowBuffer<is_binary_format>::push_time(double
data) {
return 0;
}
+template <bool is_binary_format>
+int MysqlRowBuffer<is_binary_format>::push_timev2(double data, int scale) {
+ if (is_binary_format && !_dynamic_mode) {
+ char buff[8];
+ _field_pos++;
+ float8store(buff, data);
+ return append(buff, 8);
+ }
+ // 1 for string trail, 1 for length, other for time str
+ int ret = reserve(2 + MAX_TIME_WIDTH);
+
+ if (0 != ret) {
+ LOG(ERROR) << "mysql row buffer reserve failed.";
+ return ret;
+ }
+
+ _pos = add_timev2(data, _pos, _dynamic_mode, scale);
+ return 0;
+}
template <bool is_binary_format>
template <typename DateType>
int MysqlRowBuffer<is_binary_format>::push_vec_datetime(DateType& data) {
diff --git a/be/src/util/mysql_row_buffer.h b/be/src/util/mysql_row_buffer.h
index 2df739450f..6a6c2b0ad0 100644
--- a/be/src/util/mysql_row_buffer.h
+++ b/be/src/util/mysql_row_buffer.h
@@ -72,6 +72,7 @@ public:
int push_float(float data);
int push_double(double data);
int push_time(double data);
+ int push_timev2(double data, int scale);
template <typename DateType>
int push_datetime(const DateType& data);
int push_decimal(const DecimalV2Value& data, int round_scale);
diff --git a/be/src/vec/core/call_on_type_index.h
b/be/src/vec/core/call_on_type_index.h
index bc822d9902..7e6ea4118c 100644
--- a/be/src/vec/core/call_on_type_index.h
+++ b/be/src/vec/core/call_on_type_index.h
@@ -205,6 +205,8 @@ bool call_on_index_and_data_type(TypeIndex number, F&& f) {
return f(TypePair<DataTypeNumber<Float64>, T>());
case TypeIndex::Time:
return f(TypePair<DataTypeTime, T>());
+ case TypeIndex::TimeV2:
+ return f(TypePair<DataTypeTimeV2, T>());
case TypeIndex::Decimal32:
return f(TypePair<DataTypeDecimal<Decimal32>, T>());
case TypeIndex::Decimal64:
diff --git a/be/src/vec/data_types/data_type.cpp
b/be/src/vec/data_types/data_type.cpp
index cf66b64f95..7ab7a99ca3 100644
--- a/be/src/vec/data_types/data_type.cpp
+++ b/be/src/vec/data_types/data_type.cpp
@@ -171,6 +171,8 @@ PGenericType_TypeId IDataType::get_pdata_type(const
IDataType* data_type) {
return PGenericType::TIME;
case TypeIndex::AggState:
return PGenericType::AGG_STATE;
+ case TypeIndex::TimeV2:
+ return PGenericType::TIMEV2;
default:
return PGenericType::UNKNOWN;
}
diff --git a/be/src/vec/data_types/data_type_factory.cpp
b/be/src/vec/data_types/data_type_factory.cpp
index 366cc04c89..48dbbd3270 100644
--- a/be/src/vec/data_types/data_type_factory.cpp
+++ b/be/src/vec/data_types/data_type_factory.cpp
@@ -144,9 +144,11 @@ DataTypePtr DataTypeFactory::create_data_type(const
TypeDescriptor& col_desc, bo
nested = std::make_shared<vectorized::DataTypeDateTime>();
break;
case TYPE_TIME:
- case TYPE_TIMEV2:
nested = std::make_shared<vectorized::DataTypeTime>();
break;
+ case TYPE_TIMEV2:
+ nested = std::make_shared<vectorized::DataTypeTimeV2>(col_desc.scale);
+ break;
case TYPE_DOUBLE:
nested = std::make_shared<vectorized::DataTypeFloat64>();
break;
@@ -314,7 +316,7 @@ DataTypePtr DataTypeFactory::create_data_type(const
TypeIndex& type_index, bool
nested = std::make_shared<vectorized::DataTypeQuantileStateDouble>();
break;
case TypeIndex::TimeV2:
- nested = std::make_shared<vectorized::DataTypeTime>();
+ nested = std::make_shared<vectorized::DataTypeTimeV2>();
break;
default:
DCHECK(false) << "invalid typeindex:" << getTypeName(type_index);
@@ -518,6 +520,10 @@ DataTypePtr DataTypeFactory::create_data_type(const
PColumnMeta& pcolumn) {
nested = std::make_shared<DataTypeTime>();
break;
}
+ case PGenericType::TIMEV2: {
+ nested =
std::make_shared<DataTypeTimeV2>(pcolumn.decimal_param().scale());
+ break;
+ }
case PGenericType::AGG_STATE: {
DataTypes sub_types;
for (auto child : pcolumn.children()) {
diff --git a/be/src/vec/data_types/data_type_time.cpp
b/be/src/vec/data_types/data_type_time.cpp
index 7881124f3e..d9c2af440e 100644
--- a/be/src/vec/data_types/data_type_time.cpp
+++ b/be/src/vec/data_types/data_type_time.cpp
@@ -20,6 +20,8 @@
#include "vec/data_types/data_type_time.h"
+#include <gen_cpp/data.pb.h>
+
#include <typeinfo>
#include <utility>
@@ -56,8 +58,34 @@ void DataTypeTime::to_string(const IColumn& column, size_t
row_num, BufferWritab
ostr.write(value.data(), value.size());
}
+void DataTypeTimeV2::to_pb_column_meta(PColumnMeta* col_meta) const {
+ IDataType::to_pb_column_meta(col_meta);
+ col_meta->mutable_decimal_param()->set_scale(_scale);
+}
+
MutableColumnPtr DataTypeTime::create_column() const {
return DataTypeNumberBase<Float64>::create_column();
}
+bool DataTypeTimeV2::equals(const IDataType& rhs) const {
+ return typeid(rhs) == typeid(*this);
+}
+
+std::string DataTypeTimeV2::to_string(const IColumn& column, size_t row_num)
const {
+ auto result = check_column_const_set_readability(column, row_num);
+ ColumnPtr ptr = result.first;
+ row_num = result.second;
+
+ auto value = assert_cast<const ColumnFloat64&>(*ptr).get_element(row_num);
+ return timev2_to_buffer_from_double(value, _scale);
+}
+
+void DataTypeTimeV2::to_string(const IColumn& column, size_t row_num,
BufferWritable& ostr) const {
+ std::string value = to_string(column, row_num);
+ ostr.write(value.data(), value.size());
+}
+
+MutableColumnPtr DataTypeTimeV2::create_column() const {
+ return DataTypeNumberBase<Float64>::create_column();
+}
} // namespace doris::vectorized
diff --git a/be/src/vec/data_types/data_type_time.h
b/be/src/vec/data_types/data_type_time.h
index 0bc1b1aad8..47311804e9 100644
--- a/be/src/vec/data_types/data_type_time.h
+++ b/be/src/vec/data_types/data_type_time.h
@@ -69,6 +69,42 @@ public:
DataTypeSerDeSPtr get_serde() const override { return
std::make_shared<DataTypeTimeSerDe>(); };
TypeIndex get_type_id() const override { return TypeIndex::Time; }
+ const char* get_family_name() const override { return "time"; }
};
+class DataTypeTimeV2 final : public DataTypeNumberBase<Float64> {
+public:
+ DataTypeTimeV2(int scale = 0) : _scale(scale) {
+ if (UNLIKELY(scale > 6)) {
+ LOG(FATAL) << fmt::format("Scale {} is out of bounds", scale);
+ }
+ }
+ bool equals(const IDataType& rhs) const override;
+
+ std::string to_string(const IColumn& column, size_t row_num) const
override;
+ PrimitiveType get_type_as_primitive_type() const override { return
TYPE_TIMEV2; }
+ TPrimitiveType::type get_type_as_tprimitive_type() const override {
+ return TPrimitiveType::TIMEV2;
+ }
+
+ void to_string(const IColumn& column, size_t row_num, BufferWritable&
ostr) const override;
+
+ MutableColumnPtr create_column() const override;
+
+ bool is_summable() const override { return true; }
+ bool can_be_used_in_bit_operations() const override { return true; }
+ bool can_be_used_in_boolean_context() const override { return true; }
+ bool can_be_inside_nullable() const override { return true; }
+
+ void to_pb_column_meta(PColumnMeta* col_meta) const override;
+ DataTypeSerDeSPtr get_serde() const override {
+ return std::make_shared<DataTypeTimeV2SerDe>(_scale);
+ };
+ TypeIndex get_type_id() const override { return TypeIndex::TimeV2; }
+ const char* get_family_name() const override { return "timev2"; }
+ UInt32 get_scale() const override { return _scale; }
+
+private:
+ UInt32 _scale;
+};
} // namespace doris::vectorized
diff --git a/be/src/vec/data_types/serde/data_type_time_serde.cpp
b/be/src/vec/data_types/serde/data_type_time_serde.cpp
index 46d5989a0b..4fed145c8b 100644
--- a/be/src/vec/data_types/serde/data_type_time_serde.cpp
+++ b/be/src/vec/data_types/serde/data_type_time_serde.cpp
@@ -43,5 +43,26 @@ Status DataTypeTimeSerDe::write_column_to_mysql(const
IColumn& column,
return _write_column_to_mysql(column, row_buffer, row_idx, col_const);
}
+Status DataTypeTimeV2SerDe::write_column_to_mysql(const IColumn& column,
+ MysqlRowBuffer<true>&
row_buffer, int row_idx,
+ bool col_const) const {
+ return _write_column_to_mysql(column, row_buffer, row_idx, col_const);
+}
+Status DataTypeTimeV2SerDe::write_column_to_mysql(const IColumn& column,
+ MysqlRowBuffer<false>&
row_buffer, int row_idx,
+ bool col_const) const {
+ return _write_column_to_mysql(column, row_buffer, row_idx, col_const);
+}
+template <bool is_binary_format>
+Status DataTypeTimeV2SerDe::_write_column_to_mysql(const IColumn& column,
+
MysqlRowBuffer<is_binary_format>& result,
+ int row_idx, bool
col_const) const {
+ auto& data = assert_cast<const ColumnVector<Float64>&>(column).get_data();
+ const auto col_index = index_check_const(row_idx, col_const);
+ if (UNLIKELY(0 != result.push_timev2(data[col_index], scale))) {
+ return Status::InternalError("pack mysql buffer failed.");
+ }
+ return Status::OK();
+}
} // namespace vectorized
} // namespace doris
\ No newline at end of file
diff --git a/be/src/vec/data_types/serde/data_type_time_serde.h
b/be/src/vec/data_types/serde/data_type_time_serde.h
index 3c1c14f977..c96163be55 100644
--- a/be/src/vec/data_types/serde/data_type_time_serde.h
+++ b/be/src/vec/data_types/serde/data_type_time_serde.h
@@ -41,5 +41,19 @@ private:
Status _write_column_to_mysql(const IColumn& column,
MysqlRowBuffer<is_binary_format>& result,
int row_idx, bool col_const) const;
};
+class DataTypeTimeV2SerDe : public DataTypeNumberSerDe<Float64> {
+public:
+ DataTypeTimeV2SerDe(int scale = -1) : scale(scale) {};
+ Status write_column_to_mysql(const IColumn& column, MysqlRowBuffer<true>&
row_buffer,
+ int row_idx, bool col_const) const override;
+ Status write_column_to_mysql(const IColumn& column, MysqlRowBuffer<false>&
row_buffer,
+ int row_idx, bool col_const) const override;
+
+private:
+ template <bool is_binary_format>
+ Status _write_column_to_mysql(const IColumn& column,
MysqlRowBuffer<is_binary_format>& result,
+ int row_idx, bool col_const) const;
+ int scale;
+};
} // namespace vectorized
} // namespace doris
\ No newline at end of file
diff --git a/be/src/vec/exec/format/parquet/schema_desc.cpp
b/be/src/vec/exec/format/parquet/schema_desc.cpp
index 6d87425983..f52e3921ad 100644
--- a/be/src/vec/exec/format/parquet/schema_desc.cpp
+++ b/be/src/vec/exec/format/parquet/schema_desc.cpp
@@ -237,7 +237,7 @@ TypeDescriptor
FieldDescriptor::convert_to_doris_type(tparquet::LogicalType logi
}
}
} else if (logicalType.__isset.TIME) {
- type = TypeDescriptor(TYPE_TIMEV2);
+ type = TypeDescriptor(TYPE_TIME);
} else if (logicalType.__isset.TIMESTAMP) {
type = TypeDescriptor(TYPE_DATETIMEV2);
const auto& time_unit = logicalType.TIMESTAMP.unit;
diff --git a/be/src/vec/functions/function_date_or_datetime_computation.h
b/be/src/vec/functions/function_date_or_datetime_computation.h
index e31d8f64c6..1bc5d37c53 100644
--- a/be/src/vec/functions/function_date_or_datetime_computation.h
+++ b/be/src/vec/functions/function_date_or_datetime_computation.h
@@ -281,8 +281,47 @@ struct SubtractYearsImpl :
SubtractIntervalImpl<AddYearsImpl<DateType>, DateType
}
\
};
DECLARE_DATE_FUNCTIONS(DateDiffImpl, datediff, DataTypeInt32, (ts0.daynr() -
ts1.daynr()));
-DECLARE_DATE_FUNCTIONS(TimeDiffImpl, timediff, DataTypeTime,
ts0.second_diff(ts1));
-
+// DECLARE_DATE_FUNCTIONS(TimeDiffImpl, timediff, DataTypeTime,
ts0.second_diff(ts1));
+// Expands to
+template <typename DateType1, typename DateType2>
+struct TimeDiffImpl {
+ using ArgType1 = std ::conditional_t<
+ std ::is_same_v<DateType1, DataTypeDateV2>, UInt32,
+ std ::conditional_t<std ::is_same_v<DateType1,
DataTypeDateTimeV2>, UInt64, Int64>>;
+ using ArgType2 = std ::conditional_t<
+ std ::is_same_v<DateType2, DataTypeDateV2>, UInt32,
+ std ::conditional_t<std ::is_same_v<DateType2,
DataTypeDateTimeV2>, UInt64, Int64>>;
+ using DateValueType1 = std ::conditional_t<
+ std ::is_same_v<DateType1, DataTypeDateV2>,
DateV2Value<DateV2ValueType>,
+ std ::conditional_t<std ::is_same_v<DateType1, DataTypeDateTimeV2>,
+ DateV2Value<DateTimeV2ValueType>,
VecDateTimeValue>>;
+ using DateValueType2 = std ::conditional_t<
+ std ::is_same_v<DateType2, DataTypeDateV2>,
DateV2Value<DateV2ValueType>,
+ std ::conditional_t<std ::is_same_v<DateType2, DataTypeDateTimeV2>,
+ DateV2Value<DateTimeV2ValueType>,
VecDateTimeValue>>;
+ static constexpr bool UsingTimev2 = std::is_same_v<DateType1,
DataTypeDateTimeV2> ||
+ std::is_same_v<DateType2,
DataTypeDateTimeV2> ||
+ std::is_same_v<DateType1,
DataTypeDateV2> ||
+ std::is_same_v<DateType2,
DataTypeDateV2>;
+ using ReturnType = std ::conditional_t<UsingTimev2, DataTypeTimeV2,
DataTypeTime>;
+
+ static constexpr auto name = "timediff";
+ static constexpr auto is_nullable = false;
+ static inline ReturnType ::FieldType execute(const ArgType1& t0, const
ArgType2& t1,
+ bool& is_null) {
+ const auto& ts0 = reinterpret_cast<const DateValueType1&>(t0);
+ const auto& ts1 = reinterpret_cast<const DateValueType2&>(t1);
+ is_null = !ts0.is_valid_date() || !ts1.is_valid_date();
+ if constexpr (UsingTimev2) {
+ return ts0.microsecond_diff(ts1);
+ } else {
+ return ts0.second_diff(ts1);
+ }
+ }
+ static DataTypes get_variadic_argument_types() {
+ return {std ::make_shared<DateType1>(), std
::make_shared<DateType2>()};
+ }
+};
#define TIME_DIFF_FUNCTION_IMPL(CLASS, NAME, UNIT) \
DECLARE_DATE_FUNCTIONS(CLASS, NAME, DataTypeInt64,
datetime_diff<TimeUnit::UNIT>(ts1, ts0))
diff --git a/be/src/vec/functions/function_running_difference.h
b/be/src/vec/functions/function_running_difference.h
index 248c29a9a5..73446dd2b2 100644
--- a/be/src/vec/functions/function_running_difference.h
+++ b/be/src/vec/functions/function_running_difference.h
@@ -82,8 +82,10 @@ public:
return_type = std::make_shared<DataTypeFloat64>();
} else if (which.is_decimal()) {
return_type = nested_type;
- } else if (which.is_date_time() || which.is_date_time_v2()) {
+ } else if (which.is_date_time()) {
return_type = std::make_shared<DataTypeTime>();
+ } else if (which.is_date_time_v2()) {
+ return_type = std::make_shared<DataTypeTimeV2>();
} else if (which.is_date() || which.is_date_v2()) {
return_type = std::make_shared<DataTypeInt32>();
}
diff --git a/be/src/vec/runtime/vdatetime_value.h
b/be/src/vec/runtime/vdatetime_value.h
index 891d6fb8b1..639d7171cb 100644
--- a/be/src/vec/runtime/vdatetime_value.h
+++ b/be/src/vec/runtime/vdatetime_value.h
@@ -870,6 +870,10 @@ public:
return hour() * SECOND_PER_HOUR + minute() * SECOND_PER_MINUTE +
second();
}
+ int64_t time_part_to_microsecond() const {
+ return time_part_to_seconds() * 1000 * 1000 + microsecond();
+ }
+
uint16_t year() const { return date_v2_value_.year_; }
uint8_t month() const { return date_v2_value_.month_; }
int quarter() const { return (date_v2_value_.month_ - 1) / 3 + 1; }
@@ -1065,11 +1069,27 @@ public:
return time_part_to_seconds() - rhs.time_part_to_seconds();
}
+ //only calculate the diff of dd:mm:ss.SSSSSS
+ template <typename RHS>
+ int64_t time_part_diff_microsecond(const RHS& rhs) const {
+ return time_part_to_microsecond() - rhs.time_part_to_microsecond();
+ }
+
template <typename RHS>
int64_t second_diff(const RHS& rhs) const {
return (daynr() - rhs.daynr()) * SECOND_PER_HOUR * HOUR_PER_DAY +
time_part_diff(rhs);
}
+ template <typename RHS>
+ double microsecond_diff(const RHS& rhs) const {
+ int64_t diff_m = (daynr() - rhs.daynr()) * SECOND_PER_HOUR *
HOUR_PER_DAY * 1000 * 1000 +
+ time_part_diff_microsecond(rhs);
+ if (diff_m > (int64_t)3020399 * 1000 * 1000) {
+ diff_m = (int64_t)3020399 * 1000 * 1000;
+ }
+ return diff_m;
+ }
+
bool can_cast_to_date_without_loss_accuracy() {
return this->hour() == 0 && this->minute() == 0 && this->second() == 0
&&
this->microsecond() == 0;
diff --git a/be/src/vec/sink/vmysql_result_writer.cpp
b/be/src/vec/sink/vmysql_result_writer.cpp
index fd600e0bee..6c42403343 100644
--- a/be/src/vec/sink/vmysql_result_writer.cpp
+++ b/be/src/vec/sink/vmysql_result_writer.cpp
@@ -368,9 +368,12 @@ Status
VMysqlResultWriter<is_binary_format>::_add_one_column(
if constexpr (type == TYPE_DOUBLE) {
buf_ret = rows_buffer[i].push_double(data[col_index]);
}
- if constexpr (type == TYPE_TIME || type == TYPE_TIMEV2) {
+ if constexpr (type == TYPE_TIME) {
buf_ret = rows_buffer[i].push_time(data[col_index]);
}
+ if constexpr (type == TYPE_TIMEV2) {
+ buf_ret = rows_buffer[i].push_timev2(data[col_index]);
+ }
if constexpr (type == TYPE_DATETIME) {
auto time_num = data[col_index];
VecDateTimeValue time_val = binary_cast<Int64,
VecDateTimeValue>(time_num);
diff --git a/be/test/vec/function/function_time_test.cpp
b/be/test/vec/function/function_time_test.cpp
index 45f0c96dbd..b93d66cc4a 100644
--- a/be/test/vec/function/function_time_test.cpp
+++ b/be/test/vec/function/function_time_test.cpp
@@ -859,7 +859,7 @@ TEST(VTimestampFunctionsTest, timediff_v2_test) {
{{std::string("2019-00-18"),
std::string("2019-07-18")}, Null()},
{{std::string("2019-07-18"),
std::string("2019-07-00")}, Null()}};
- check_function<DataTypeTime, true>(func_name, input_types, data_set);
+ check_function<DataTypeTimeV2, true>(func_name, input_types, data_set);
}
{
@@ -867,11 +867,12 @@ TEST(VTimestampFunctionsTest, timediff_v2_test) {
DataSet data_set = {
{{std::string("2019-07-18 00:00:00"), std::string("2019-07-18
00:00:00")}, 0.0},
- {{std::string("2019-07-18 00:00:10"), std::string("2019-07-18
00:00:00")}, 10.0},
+ {{std::string("2019-07-18 00:00:10"), std::string("2019-07-18
00:00:00")},
+ 10000000.0},
{{std::string("2019-00-18 00:00:00"), std::string("2019-07-18
00:00:00")}, Null()},
{{std::string("2019-07-18 00:00:00"), std::string("2019-07-00
00:00:00")}, Null()}};
- check_function<DataTypeTime, true>(func_name, input_types, data_set);
+ check_function<DataTypeTimeV2, true>(func_name, input_types, data_set);
}
}
diff --git
a/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java
b/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java
index dc36a9ab70..41cd59d474 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java
@@ -143,6 +143,8 @@ public class ScalarType extends Type {
return createDecimalType(precision, scale);
case DATETIMEV2:
return createDatetimeV2Type(scale);
+ case TIMEV2:
+ return createTimeV2Type(scale);
default:
return createType(type);
}
@@ -687,6 +689,12 @@ public class ScalarType extends Type {
scalarType.setPrecision(precision);
break;
}
+ case TIMEV2: {
+ Preconditions.checkArgument(precision >= scale);
+ scalarType.setScale(scale);
+ scalarType.setPrecision(precision);
+ break;
+ }
default:
break;
}
@@ -867,7 +875,10 @@ public class ScalarType extends Type {
return false;
}
ScalarType other = (ScalarType) o;
- if ((this.isDatetimeV2() && other.isDatetimeV2()) || (this.isTimeV2()
&& other.isTimeV2())) {
+ if ((this.isDatetimeV2() && other.isDatetimeV2())) {
+ return this.decimalScale() == other.decimalScale();
+ }
+ if (this.isTimeV2() && other.isTimeV2()) {
return this.decimalScale() == other.decimalScale();
}
if (type.isDecimalV3Type() && other.isDecimalV3()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
index d56c398836..705a45ca37 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
@@ -2457,7 +2457,7 @@ public abstract class Expr extends TreeNode<Expr>
implements ParseNode, Cloneabl
}
}
- protected Type getActualScalarType(Type originType) {
+ protected Type getActualScalarType(Type originType) {
if (originType.getPrimitiveType() == PrimitiveType.DECIMAL32) {
return Type.DECIMAL32;
} else if (originType.getPrimitiveType() == PrimitiveType.DECIMAL64) {
@@ -2472,6 +2472,8 @@ public abstract class Expr extends TreeNode<Expr>
implements ParseNode, Cloneabl
return Type.CHAR;
} else if (originType.getPrimitiveType() == PrimitiveType.DECIMALV2) {
return Type.MAX_DECIMALV2_TYPE;
+ } else if (originType.getPrimitiveType() == PrimitiveType.TIMEV2) {
+ return Type.TIMEV2;
}
return originType;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
index 24169eccf8..4e57ee9df0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
@@ -1559,6 +1559,13 @@ public class FunctionCallExpr extends Expr {
}
if (fn.getFunctionName().getFunction().equals("timediff")) {
fn.getReturnType().getPrimitiveType().setTimeType();
+ ScalarType left = (ScalarType) argTypes[0];
+ ScalarType right = (ScalarType) argTypes[1];
+ if (left.isDatetimeV2() || right.isDatetimeV2() || left.isDateV2()
|| right.isDateV2()) {
+ Type ret =
ScalarType.createTimeV2Type(Math.max(left.getScalarScale(),
right.getScalarScale()));
+ fn.setReturnType(ret);
+ fn.getReturnType().getPrimitiveType().setTimeType();
+ }
}
if (fnName.getFunction().equalsIgnoreCase("map")) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/TimeDiff.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/TimeDiff.java
index 8004e254a2..2cff7c8886 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/TimeDiff.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/TimeDiff.java
@@ -21,6 +21,7 @@ import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import
org.apache.doris.nereids.trees.expressions.functions.PropagateNullableOnDateLikeV2Args;
+import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral;
import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.DateTimeType;
@@ -28,6 +29,7 @@ import org.apache.doris.nereids.types.DateTimeV2Type;
import org.apache.doris.nereids.types.DateV2Type;
import org.apache.doris.nereids.types.TimeType;
import org.apache.doris.nereids.types.TimeV2Type;
+import org.apache.doris.nereids.types.coercion.AbstractDataType;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
@@ -46,8 +48,7 @@ public class TimeDiff extends ScalarFunction
FunctionSignature.ret(TimeType.INSTANCE).args(DateTimeType.INSTANCE,
DateTimeType.INSTANCE),
FunctionSignature.ret(TimeV2Type.INSTANCE).args(DateTimeV2Type.SYSTEM_DEFAULT,
DateV2Type.INSTANCE),
FunctionSignature.ret(TimeV2Type.INSTANCE).args(DateV2Type.INSTANCE,
DateTimeV2Type.SYSTEM_DEFAULT),
-
FunctionSignature.ret(TimeV2Type.INSTANCE).args(DateV2Type.INSTANCE,
DateV2Type.INSTANCE)
- );
+
FunctionSignature.ret(TimeV2Type.INSTANCE).args(DateV2Type.INSTANCE,
DateV2Type.INSTANCE));
/**
* constructor with 2 arguments.
@@ -74,4 +75,41 @@ public class TimeDiff extends ScalarFunction
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
return visitor.visitTimeDiff(this, context);
}
+
+ @Override
+ public FunctionSignature computeSignature(FunctionSignature signature) {
+ signature = super.computeSignature(signature);
+ boolean useTimev2 = false;
+ int scale = 0;
+ if (getArgument(0).getDataType() instanceof DateTimeV2Type) {
+ DateTimeV2Type left = (DateTimeV2Type)
getArgument(0).getDataType();
+ scale = Math.max(scale, left.getScale());
+ useTimev2 = true;
+ }
+ if (getArgument(1).getDataType() instanceof DateTimeV2Type) {
+ DateTimeV2Type right = (DateTimeV2Type)
getArgument(1).getDataType();
+ scale = Math.max(scale, right.getScale());
+ useTimev2 = true;
+ }
+ if (useTimev2) {
+ signature = signature.withReturnType(TimeV2Type.of(scale));
+ }
+ return signature;
+ }
+
+ @Override
+ public List<AbstractDataType> expectedInputTypes() {
+ FunctionSignature signature = getSignature();
+ if (getArgument(0) instanceof StringLikeLiteral) {
+ StringLikeLiteral str = (StringLikeLiteral) getArgument(0);
+ DateTimeV2Type left =
DateTimeV2Type.forTypeFromString(str.getStringValue());
+ signature = signature.withArgumentType(0, left);
+ }
+ if (getArgument(1) instanceof StringLikeLiteral) {
+ StringLikeLiteral str = (StringLikeLiteral) getArgument(1);
+ DateTimeV2Type right =
DateTimeV2Type.forTypeFromString(str.getStringValue());
+ signature = signature.withArgumentType(1, right);
+ }
+ return signature.argumentsTypes;
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/TimeV2Type.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/TimeV2Type.java
index f798a9b3f7..ec625d0cd1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/TimeV2Type.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/TimeV2Type.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.types;
+import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.nereids.types.coercion.PrimitiveType;
@@ -28,13 +29,26 @@ public class TimeV2Type extends PrimitiveType {
public static final TimeV2Type INSTANCE = new TimeV2Type();
private static final int WIDTH = 8;
+ private final int scale;
+
+ private TimeV2Type(int scale) {
+ this.scale = scale;
+ }
private TimeV2Type() {
+ scale = 0;
}
@Override
public Type toCatalogDataType() {
- return Type.TIMEV2;
+ return ScalarType.createTimeV2Type(scale);
+ }
+
+ /**
+ * create TimeV2Type from scale
+ */
+ public static TimeV2Type of(int scale) {
+ return new TimeV2Type(scale);
}
@Override
@@ -46,4 +60,8 @@ public class TimeV2Type extends PrimitiveType {
public int width() {
return WIDTH;
}
+
+ public int getScale() {
+ return scale;
+ }
}
diff --git a/gensrc/proto/types.proto b/gensrc/proto/types.proto
index 7b74f6a804..af0f1fcc1d 100644
--- a/gensrc/proto/types.proto
+++ b/gensrc/proto/types.proto
@@ -109,6 +109,7 @@ message PGenericType {
QUANTILE_STATE = 34;
TIME = 35;
AGG_STATE = 36;
+ TIMEV2 = 37;
UNKNOWN = 999;
}
required TypeId id = 2;
diff --git a/regression-test/data/correctness/test_time_diff_microseconds.out
b/regression-test/data/correctness/test_time_diff_microseconds.out
new file mode 100644
index 0000000000..ce4c6c10f5
--- /dev/null
+++ b/regression-test/data/correctness/test_time_diff_microseconds.out
@@ -0,0 +1,29 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !select1 --
+67:19:00.0000
+24:00:00.5140
+-437:00:00.7683
+
+-- !select2 --
+48:00:00
+
+-- !select3 --
+48:00:00.114514
+
+-- !select4 --
+48:00:00.11400
+
+-- !select5 --
+67:19:00.0000
+24:00:00.5140
+-437:00:00.7683
+
+-- !select6 --
+48:00:00
+
+-- !select7 --
+48:00:00.114514
+
+-- !select8 --
+48:00:00.11400
+
diff --git a/regression-test/data/nereids_p0/test_char_implicit_cast.out
b/regression-test/data/nereids_p0/test_char_implicit_cast.out
index 59f5d47377..599dd6500d 100644
--- a/regression-test/data/nereids_p0/test_char_implicit_cast.out
+++ b/regression-test/data/nereids_p0/test_char_implicit_cast.out
@@ -6,10 +6,10 @@
7
-- !test_timediff_varchar --
--24:00:00
+-96:00:00
--- !test_timediff_char --
--24:00:00
+-- !test_timediff_varcharv2 --
+-96:00:00.3337
-- !test_money_format_varchar --
123,456.00
diff --git
a/regression-test/suites/correctness/test_time_diff_microseconds.groovy
b/regression-test/suites/correctness/test_time_diff_microseconds.groovy
new file mode 100644
index 0000000000..e754515250
--- /dev/null
+++ b/regression-test/suites/correctness/test_time_diff_microseconds.groovy
@@ -0,0 +1,95 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_time_diff_microseconds") {
+ sql """ DROP TABLE IF EXISTS tbl_time """
+ sql """
+ CREATE TABLE IF NOT EXISTS tbl_time (
+ `id` int(11) ,
+ `t1` DATETIMEV2(3) ,
+ `t2` DATETIMEV2(4)
+ )
+ UNIQUE KEY(`id`)
+ DISTRIBUTED BY HASH(`id`) BUCKETS 10
+ PROPERTIES (
+ "enable_unique_key_merge_on_write" = "true",
+ "replication_num" = "1"
+ );
+ """
+ sql """
+ INSERT INTO tbl_time VALUES(1,'0001-01-03 19:19:00.000000','0001-01-01
00:00:00.0000');
+ """
+
+ sql """
+ INSERT INTO tbl_time VALUES(2,'0001-01-02 00:00:00.514514','0001-01-01
00:00:00.000000');
+ """
+
+ sql """
+ INSERT INTO tbl_time VALUES(3,'0001-01-03 19:00:00.123123','0001-01-22
00:00:00.891312');
+ """
+
+ sql """set enable_nereids_planner=false"""
+
+ qt_select1 """
+ select timediff(t1,t2) from tbl_time order by id
+ """
+
+ qt_select2 """
+ select timediff(
+ cast('0001-01-03 11:45:14' as Date ) ,
+ cast('0001-01-01 00:00:00.000000' as Date));
+ """
+
+ qt_select3 """
+ select timediff(
+ cast('0001-01-03 00:00:00.114514' as Datetimev2(6) ) ,
+ cast('0001-01-01 00:00:00.000000' as Datetimev2(6) ));
+ """
+
+
+ qt_select4 """
+ select timediff(
+ cast('0001-01-03 00:00:00.114514' as Datetimev2(3) ) ,
+ cast('0001-01-01 00:00:00.000000' as Datetimev2(5) ));
+ """
+
+ sql """set enable_nereids_planner=true """
+ sql """set enable_fallback_to_original_planner=false"""
+
+ qt_select5 """
+ select timediff(t1,t2) from tbl_time order by id
+ """
+
+ qt_select6 """
+ select timediff(
+ cast('0001-01-03 11:45:14' as Date ) ,
+ cast('0001-01-01 00:00:00.000000' as Date));
+ """
+
+ qt_select7 """
+ select timediff(
+ cast('0001-01-03 00:00:00.114514' as Datetimev2(6) ) ,
+ cast('0001-01-01 00:00:00.000000' as Datetimev2(6) ));
+ """
+
+
+ qt_select8 """
+ select timediff(
+ cast('0001-01-03 00:00:00.114514' as Datetimev2(3) ) ,
+ cast('0001-01-01 00:00:00.000000' as Datetimev2(5) ));
+ """
+}
diff --git a/regression-test/suites/nereids_p0/test_char_implicit_cast.groovy
b/regression-test/suites/nereids_p0/test_char_implicit_cast.groovy
index b7e23812e8..415a808365 100644
--- a/regression-test/suites/nereids_p0/test_char_implicit_cast.groovy
+++ b/regression-test/suites/nereids_p0/test_char_implicit_cast.groovy
@@ -20,8 +20,8 @@ suite("test_char_implicit_cast") {
sql "SET enable_fallback_to_original_planner=false"
qt_test_dayofweek_varchar 'select dayofweek("2012-12-01");'
qt_test_dayofweek_char 'select dayofweek(cast("2012-12-01" as char(16)));'
- qt_test_timediff_varchar 'select timediff("2010-01-01 01:00:00",
"2010-01-02 01:00:00");'
- qt_test_timediff_char 'select timediff("2010-01-01 01:00:00",
cast("2010-01-02 01:00:00" as char));'
+ qt_test_timediff_varchar 'select timediff("2010-01-01 01:00:00",
"2010-01-05 01:00:00");'
+ qt_test_timediff_varcharv2 'select timediff("2010-01-01 01:00:00.123",
"2010-01-05 01:00:00.4567");'
qt_test_money_format_varchar 'select money_format("123456");'
qt_test_money_format_char 'select money_format(cast("123456" as char));'
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
