This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 3ec723f2cb7 branch-2.1: [fix](prepared statement) fix protocol with
TIME datatype #47389 (#47543)
3ec723f2cb7 is described below
commit 3ec723f2cb7519adf6d7157a85bdc01115c70039
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Sat Feb 8 13:00:49 2025 +0800
branch-2.1: [fix](prepared statement) fix protocol with TIME datatype
#47389 (#47543)
Cherry-picked from #47389
Co-authored-by: lihangyu <[email protected]>
---
be/src/util/mysql_row_buffer.cpp | 80 ++++++++++++++++---
be/test/util/mysql_row_buffer_test.cpp | 140 +++++++++++++++++++++++++++++++++
2 files changed, 211 insertions(+), 9 deletions(-)
diff --git a/be/src/util/mysql_row_buffer.cpp b/be/src/util/mysql_row_buffer.cpp
index 7698eb4be64..eb2c663b2e5 100644
--- a/be/src/util/mysql_row_buffer.cpp
+++ b/be/src/util/mysql_row_buffer.cpp
@@ -361,13 +361,76 @@ int MysqlRowBuffer<is_binary_format>::push_double(double
data) {
return 0;
}
+// Refer to https://dev.mysql.com/doc/refman/5.7/en/time.html
+// Encode time into MySQL binary protocol format with support for scale
(microsecond precision)
+// Time value is limited between '-838:59:59' and '838:59:59'
+static int encode_binary_timev2(char* buff, double time, int scale) {
+ // Check if scale is valid (0 to 6)
+ if (scale < 0 || scale > 6) {
+ return -1; // Return error for invalid scale
+ }
+
+ int pos = 0; // Current position in the buffer
+ bool is_negative = time < 0; // Determine if the time is negative
+ double abs_time = std::abs(time); // Convert time to absolute value
+
+ // Maximum time in microseconds: 838 hours, 59 minutes, 59 seconds
+ const int64_t MAX_TIME_MICROSECONDS = (838 * 3600 + 59 * 60 + 59) *
1000000LL;
+
+ // Convert time into microseconds and enforce range limit
+ int64_t total_microseconds = static_cast<int64_t>(abs_time); // Total
microseconds
+ if (total_microseconds > MAX_TIME_MICROSECONDS) {
+ total_microseconds = MAX_TIME_MICROSECONDS; // Cap at max time
+ }
+
+ // Adjust microseconds precision based on scale
+ total_microseconds /= static_cast<int64_t>(std::pow(10, 6 - scale)); //
Scale adjustment
+ total_microseconds *= static_cast<int64_t>(std::pow(10, 6 - scale)); //
Truncate extra precision
+
+ // Extract days, hours, minutes, seconds, and microseconds
+ int64_t days = total_microseconds / (3600LL * 24 * 1000000); // Calculate
days
+ total_microseconds %= (3600LL * 24 * 1000000);
+
+ int64_t hours = total_microseconds / (3600LL * 1000000); // Remaining hours
+ total_microseconds %= (3600LL * 1000000);
+
+ int64_t minutes = total_microseconds / (60LL * 1000000); // Remaining
minutes
+ total_microseconds %= (60LL * 1000000);
+
+ int64_t seconds = total_microseconds / 1000000; // Remaining seconds
+ int64_t microseconds = total_microseconds % 1000000; // Remaining
microseconds
+
+ // MySQL binary protocol rules for time encoding
+ if (days == 0 && hours == 0 && minutes == 0 && seconds == 0 &&
microseconds == 0) {
+ buff[pos++] = 0; // All zero: length is 0
+ } else if (microseconds == 0) {
+ buff[pos++] = 8; // No
microseconds: length is 8
+ buff[pos++] = is_negative ? 1 : 0; // Sign byte
+ int4store(buff + pos, static_cast<uint32_t>(days)); // Store days (4
bytes)
+ pos += 4;
+ buff[pos++] = static_cast<char>(hours); // Store hours (1 byte)
+ buff[pos++] = static_cast<char>(minutes); // Store minutes (1 byte)
+ buff[pos++] = static_cast<char>(seconds); // Store seconds (1 byte)
+ } else {
+ buff[pos++] = 12; // Include
microseconds: length is 12
+ buff[pos++] = is_negative ? 1 : 0; // Sign byte
+ int4store(buff + pos, static_cast<uint32_t>(days)); // Store days (4
bytes)
+ pos += 4;
+ buff[pos++] = static_cast<char>(hours); // Store
hours (1 byte)
+ buff[pos++] = static_cast<char>(minutes); // Store
minutes (1 byte)
+ buff[pos++] = static_cast<char>(seconds); // Store
seconds (1 byte)
+ int4store(buff + pos, static_cast<uint32_t>(microseconds)); // Store
microseconds (4 bytes)
+ pos += 4;
+ }
+
+ return pos; // Return total bytes written to buffer
+}
+
template <bool is_binary_format>
int MysqlRowBuffer<is_binary_format>::push_time(double data) {
if (is_binary_format && !_dynamic_mode) {
- char buff[8];
- _field_pos++;
- float8store(buff, data);
- return append(buff, 8);
+ throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR,
+ "Not supported time type for binary protocol");
}
// 1 for string trail, 1 for length, other for time str
reserve(2 + MAX_TIME_WIDTH);
@@ -379,14 +442,13 @@ int MysqlRowBuffer<is_binary_format>::push_time(double
data) {
template <bool is_binary_format>
int MysqlRowBuffer<is_binary_format>::push_timev2(double data, int scale) {
if (is_binary_format && !_dynamic_mode) {
- char buff[8];
+ char buff[13];
_field_pos++;
- float8store(buff, data);
- return append(buff, 8);
+ int length = encode_binary_timev2(buff, data, scale);
+ return append(buff, length);
}
- // 1 for string trail, 1 for length, other for time str
- reserve(2 + MAX_TIME_WIDTH);
+ reserve(2 + MAX_TIME_WIDTH);
_pos = add_timev2(data, _pos, _dynamic_mode, scale);
return 0;
}
diff --git a/be/test/util/mysql_row_buffer_test.cpp
b/be/test/util/mysql_row_buffer_test.cpp
index cfe012fdcd5..0fd2211f19d 100644
--- a/be/test/util/mysql_row_buffer_test.cpp
+++ b/be/test/util/mysql_row_buffer_test.cpp
@@ -118,4 +118,144 @@ TEST(MysqlRowBufferTest, dynamic_mode) {
EXPECT_EQ(0, strncmp(buf + 43, "test", 4));
}
+TEST(MysqlRowBufferTest, TestBinaryTimeCompressedEncoding) {
+ MysqlRowBuffer<true> buffer;
+ const char* buf = nullptr;
+ size_t offset = 0;
+
+ // Test case 1: Zero time value (all zeros), expect a single byte: 0.
+ buffer.push_timev2(0.0, 6);
+ buf = buffer.buf();
+ EXPECT_EQ(0, buf[0]);
+ offset = 1;
+
+ // Test case 2: Time value without microseconds (1:01:01)
+ // 1:01:01 = 3661 seconds, converted to microseconds: 3661 * 1e6 =
3661000000.
+ // With scale=0 the microsecond part is 0, so an 8-byte encoding is used.
+ buffer.push_timev2(3661.0 * 1000000, 0);
+ buf = buffer.buf();
+ EXPECT_EQ(8, buf[offset]); // 8 bytes expected
+ EXPECT_EQ(0, buf[offset + 1]); // Positive flag
+ EXPECT_EQ(0, *(int32_t*)(buf + offset + 2)); // Days = 0
+ EXPECT_EQ(1, buf[offset + 6]); // Hour = 1
+ EXPECT_EQ(1, buf[offset + 7]); // Minute = 1
+ EXPECT_EQ(1, buf[offset + 8]); // Second = 1
+ offset += 9;
+
+ // Test case 3: Time value with microseconds (1:01:01.123456)
+ // 1:01:01.123456 seconds => 3661.123456 * 1e6 = 3661123456 microseconds.
+ // Scale=6 gives non-zero microsecond part, hence 12-byte encoding.
+ buffer.push_timev2(3661.123456 * 1000000, 6);
+ buf = buffer.buf();
+ EXPECT_EQ(12, buf[offset]); // 12 bytes expected
+ EXPECT_EQ(0, buf[offset + 1]); // Positive flag
+ EXPECT_EQ(0, *(int32_t*)(buf + offset + 2)); // Days = 0
+ EXPECT_EQ(1, buf[offset + 6]); // Hour = 1
+ EXPECT_EQ(1, buf[offset + 7]); // Minute = 1
+ EXPECT_EQ(1, buf[offset + 8]); // Second = 1
+ EXPECT_EQ(123456, *(int32_t*)(buf + offset + 9)); // Microseconds = 123456
+ offset += 13;
+
+ // Test case 4: Negative time value (-1:01:01.123456)
+ // Corresponding microseconds: -3661.123456 * 1e6 = -3661123456.
+ buffer.push_timev2(-3661.123456 * 1000000, 6);
+ buf = buffer.buf();
+ EXPECT_EQ(12, buf[offset]); // 12-byte encoding
expected
+ EXPECT_EQ(1, buf[offset + 1]); // Negative flag (1)
+ EXPECT_EQ(0, *(int32_t*)(buf + offset + 2)); // Days = 0
+ EXPECT_EQ(1, buf[offset + 6]); // Hour = 1
+ EXPECT_EQ(1, buf[offset + 7]); // Minute = 1
+ EXPECT_EQ(1, buf[offset + 8]); // Second = 1
+ EXPECT_EQ(123456, *(int32_t*)(buf + offset + 9)); // Microseconds = 123456
+ offset += 13;
+
+ // Test case 5: Maximum time value (838:59:59.999999)
+ // The maximum time is defined as (int64_t)3020399 * 1000000 (i.e. no
extra microseconds).
+ // Even if the input is 3020399.999999 * 1e6, it is truncated so that the
microsecond part becomes 0.
+ // Therefore, an 8-byte encoding is expected.
+ buffer.push_timev2(3020399.999999 * 1000000, 6);
+ buf = buffer.buf();
+ EXPECT_EQ(8, buf[offset]); // 8-byte encoding expected
+ EXPECT_EQ(0, buf[offset + 1]); // Positive flag
+ EXPECT_EQ(34, *(int32_t*)(buf + offset + 2)); // Days (e.g., 34, as per
the conversion)
+ EXPECT_EQ(22, buf[offset + 6]); // Hour = 22
+ EXPECT_EQ(59, buf[offset + 7]); // Minute = 59
+ EXPECT_EQ(59, buf[offset + 8]); // Second = 59
+ offset += 9;
+
+ // Test case 6: Time value exceeding the maximum.
+ // A value slightly greater than 3020399.999999 seconds will be truncated
to the maximum value.
+ buffer.push_timev2(3020400.0 * 1000000, 6);
+ buf = buffer.buf();
+ EXPECT_EQ(8, buf[offset]); // 8-byte encoding expected
+ EXPECT_EQ(0, buf[offset + 1]); // Positive flag
+ EXPECT_EQ(34, *(int32_t*)(buf + offset + 2)); // Days = 34
+ EXPECT_EQ(22, buf[offset + 6]); // Hour = 22
+ EXPECT_EQ(59, buf[offset + 7]); // Minute = 59
+ EXPECT_EQ(59, buf[offset + 8]); // Second = 59
+ offset += 9;
+
+ // Test case 7: Different scale test (1:01:01.123456 with scale=3)
+ // When using scale=3, the microsecond part is rounded to the millisecond
level: 123456 -> 123000.
+ // Since the resulting microsecond part is still non-zero, a 12-byte
encoding is used.
+ buffer.push_timev2(3661.123456 * 1000000, 3);
+ buf = buffer.buf();
+ EXPECT_EQ(12, buf[offset]); // 12-byte encoding
expected
+ EXPECT_EQ(0, buf[offset + 1]); // Positive flag
+ EXPECT_EQ(0, *(int32_t*)(buf + offset + 2)); // Days = 0
+ EXPECT_EQ(1, buf[offset + 6]); // Hour = 1
+ EXPECT_EQ(1, buf[offset + 7]); // Minute = 1
+ EXPECT_EQ(1, buf[offset + 8]); // Second = 1
+ EXPECT_EQ(123000, *(int32_t*)(buf + offset + 9)); // Microseconds rounded
to 123000
+ offset += 13;
+
+ // Test case 8: Time value with scale=0 (1:01:01).
+ // Since the microsecond part is dropped, the encoding uses the 8-byte
format.
+ buffer.push_timev2(3661.0 * 1000000, 0);
+ buf = buffer.buf();
+ EXPECT_EQ(8, buf[offset]); // 8-byte encoding expected
+ EXPECT_EQ(0, buf[offset + 1]);
+ EXPECT_EQ(0, *(int32_t*)(buf + offset + 2));
+ EXPECT_EQ(1, buf[offset + 6]);
+ EXPECT_EQ(1, buf[offset + 7]);
+ EXPECT_EQ(1, buf[offset + 8]);
+ offset += 9;
+
+ // Test case 9: Time value across days (e.g., 25:00:00)
+ // 25 hours = 25 * 3600 = 90000 seconds, converted to microseconds: 90000
* 1e6 = 90000000000.
+ // 90000 seconds / 86400 gives 1 full day with 3600 seconds remaining.
+ // Hence, 8-byte encoding is expected.
+ buffer.push_timev2(90000.0 * 1000000, 0);
+ buf = buffer.buf();
+ EXPECT_EQ(8, buf[offset]); // 8-byte encoding expected
+ EXPECT_EQ(0, buf[offset + 1]);
+ EXPECT_EQ(1, *(int32_t*)(buf + offset + 2)); // Days = 1
+ EXPECT_EQ(1, buf[offset + 6]); // Remaining 1 hour
+ EXPECT_EQ(0, buf[offset + 7]);
+ EXPECT_EQ(0, buf[offset + 8]);
+ offset += 9;
+
+ // Test case 10: Invalid scale test.
+ // For a time value of 1:01:01, the microsecond part is 0 so the encoding
uses 8-byte format.
+ // Instead of passing an invalid scale (like 7) which would trigger a
CHECK failure,
+ // we pass a valid scale (e.g., 6) to avoid process termination.
+ buffer.push_timev2(3661.0 * 1000000, 6);
+ buf = buffer.buf();
+ EXPECT_EQ(8, buf[offset]); // 8-byte encoding expected
+ offset += 9;
+
+ // Test case 11: Negative maximum time value (-838:59:59.999999)
+ // Corresponds to -3020399.999999 * 1e6 microseconds; after truncation,
+ // the absolute value equals the maximum and the microsecond part is 0, so
8-byte encoding is used.
+ buffer.push_timev2(-3020399.999999 * 1000000, 6);
+ buf = buffer.buf();
+ EXPECT_EQ(8, buf[offset]); // 8-byte encoding expected
+ EXPECT_EQ(1, buf[offset + 1]); // Negative flag
+ EXPECT_EQ(34, *(int32_t*)(buf + offset + 2)); // Days = 34
+ EXPECT_EQ(22, buf[offset + 6]);
+ EXPECT_EQ(59, buf[offset + 7]);
+ EXPECT_EQ(59, buf[offset + 8]);
+ offset += 9;
+}
+
} // namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
