github-actions[bot] commented on code in PR #68297:
URL: https://github.com/apache/doris/pull/68297#discussion_r4063528101
##########
be/src/core/data_type_serde/data_type_varbinary_serde.cpp:
##########
@@ -301,6 +305,82 @@ Status
DataTypeVarbinarySerDe::deserialize_one_cell_from_json(IColumn& column, S
return Status::OK();
}
+Status DataTypeVarbinarySerDe::from_string(StringRef& str, IColumn& column,
+ const FormatOptions& options) const
{
+ // Partition structs use the same hex representation as nested VARBINARY
output. Decode it
+ // before appending so arbitrary bytes survive JSON transport instead of
becoming NULL.
+ if (str.size < 2 || str.data[0] != '0' || str.data[1] != 'x' || (str.size
- 2) % 2 != 0 ||
+ str.size - 2 > std::numeric_limits<int>::max()) {
+ return Status::InvalidArgument("Invalid VARBINARY hex representation");
+ }
+ // The INT_MAX guard also makes narrowing to the decoder's 32-bit offset
type safe.
+ const auto hex_size = cast_set<ColumnString::Offset>(str.size - 2);
+ std::string bytes(hex_size / 2, '\0');
+ if (string_hex::hex_decode(str.data + 2, hex_size, bytes.data()) !=
bytes.size()) {
+ return Status::InvalidArgument("Invalid VARBINARY hex representation");
+ }
+ assert_cast<ColumnVarbinary&>(column).insert_data(bytes.data(),
bytes.size());
+ return Status::OK();
+}
+
+Status DataTypeVarbinarySerDe::deserialize_one_cell_from_hive_text(
+ IColumn& column, Slice& slice, const FormatOptions& options,
+ int hive_text_complex_type_delimiter_level) const {
+ // Hive LazyBinary uses lenient Base64 (including URL-safe letters and
whitespace),
+ // falling back to the original bytes for non-Base64 input or an empty
decoding.
+ // Keep this separate from JSON/CSV: those formats do not share Hive's
encoding contract.
+ std::string encoded;
+ encoded.reserve(slice.size);
+ bool padding = false;
+ for (size_t i = 0; i < slice.size; ++i) {
+ const char c = slice.data[i];
+ if (c == ' ' || c == '\t' || c == '\r' || c == '\n') {
+ continue;
+ }
+ if (c == '=') {
+ padding = true;
+ continue;
+ }
+ if (!((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' &&
c <= '9') ||
+ c == '+' || c == '/' || c == '-' || c == '_')) {
+ return deserialize_one_cell_from_json(column, slice, options);
+ }
+ if (!padding) {
+ encoded.push_back(c == '-' ? '+' : c == '_' ? '/' : c);
+ }
+ }
+ // Commons Codec ignores a trailing sextet and accepts omitted padding.
+ if (encoded.size() % 4 == 1) {
+ encoded.pop_back();
+ }
+ encoded.append((4 - encoded.size() % 4) % 4, '=');
+ std::string decoded;
+ if (!base64_decode(encoded, &decoded) || decoded.empty()) {
+ return deserialize_one_cell_from_json(column, slice, options);
+ }
+ assert_cast<ColumnVarbinary&>(column).insert_data(decoded.data(),
decoded.size());
+ return Status::OK();
+}
+
+Status DataTypeVarbinarySerDe::deserialize_column_from_hive_text_vector(
+ IColumn& column, std::vector<Slice>& slices, uint64_t*
num_deserialized,
+ const FormatOptions& options, int
hive_text_complex_type_delimiter_level) const {
+ DESERIALIZE_COLUMN_FROM_HIVE_TEXT_VECTOR()
+ return Status::OK();
+}
+
+Status DataTypeVarbinarySerDe::serialize_one_cell_to_hive_text(
+ const IColumn& column, int64_t row_num, BufferWritable& bw,
FormatOptions& options,
+ int hive_text_complex_type_delimiter_level) const {
+ auto [data_column, data_row] = check_column_const_set_readability(column,
row_num);
+ const auto value = assert_cast<const
ColumnVarbinary&>(*data_column).get_data_at(data_row);
+ // Encoding is required on write as well, or a Hive reader will
reinterpret binary bytes.
+ std::string encoded;
+ base64_encode(value.to_string(), &encoded);
Review Comment:
[P1] Match current Hive's unpadded Base64 before delimiter splitting
Current Hive writes BINARY text with unpadded Base64, but `base64_encode`
emits `=` padding and this path writes it without escaping. A valid table with
`field.delim='='` (or `=` at a nested separator level) turns `0x00` into
`AA==`; the text writer emits it verbatim and the readers split the padding
bytes into extra fields/elements, silently corrupting the row. The decoder
already restores omitted padding, so please strip trailing `=` to match the
current Hive form and add scalar/nested round-trip coverage with `=` configured
as the separator.
##########
be/src/exprs/function/cast/cast_to_datetimev2_impl.hpp:
##########
@@ -650,24 +651,47 @@ inline bool
CastToDatetimeV2::from_string_strict_mode_internal(
SET_PARAMS_RET_FALSE_IFN((consume_digit<UInt32, 2>(ptr, end,
part[0])),
"invalid hour offset '{}'",
std::string {ptr, end});
}
- SET_PARAMS_RET_FALSE_IFN(part[0] <= 14, "invalid hour offset
'{}'", part[0]);
+ SET_PARAMS_RET_FALSE_IFN(
Review Comment:
[P2] Keep FE and BE TIMESTAMPTZ offset ranges aligned
This widens BE parsing to every offset below 24 hours, but FE literal
coercion still uses Java `ZoneOffset`, whose range ends at `+/-18:00`. An
explicit cast that reaches BE can therefore accept `2020-01-01 19:00:00+19:00`,
while `ts = '2020-01-01 19:00:00+19:00'` cannot build a `TimestampTzLiteral`
and falls back to the narrower DATETIMEV2 parser, yielding an error or NULL
instead of comparing the same instant. Please use one exact range in FE and BE,
and cover `+19:00`/`-18:00:01` across explicit casts, implicit comparisons, and
fold modes.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
