hubgeter commented on code in PR #69512:
URL: https://github.com/apache/doris/pull/69512#discussion_r4357506572
##########
be/src/core/data_type/data_type_nullable.cpp:
##########
@@ -103,11 +103,60 @@ char* DataTypeNullable::serialize(const IColumn& column,
char* buf, int be_exec_
const char* DataTypeNullable::deserialize(const char* buf, MutableColumnPtr*
column,
int be_exec_version) const {
+ return deserialize(buf, nullptr, column, be_exec_version);
+}
+
+const char* DataTypeNullable::deserialize(const char* buf, const char* end,
+ MutableColumnPtr* column, int
be_exec_version) const {
auto* origin_column = column->get();
+ constexpr size_t header_size = sizeof(bool) + 2 * sizeof(size_t);
+ if (end != nullptr) {
+ if (end < buf || static_cast<size_t>(end - buf) < header_size) {
+ throw Exception(ErrorCode::INVALID_ARGUMENT,
+ "Malformed nullable column: missing serialized
header");
+ }
+ uint8_t is_const_flag = unaligned_load<uint8_t>(buf);
+ if (is_const_flag > 1) {
+ throw Exception(ErrorCode::INVALID_ARGUMENT,
+ "Malformed nullable column: invalid const flag
{}", is_const_flag);
+ }
+ bool is_const = is_const_flag != 0;
+ size_t row_num = unaligned_load<size_t>(buf + sizeof(bool));
+ size_t saved_num = unaligned_load<size_t>(buf + sizeof(bool) +
sizeof(size_t));
+ if (saved_num != (is_const ? 1 : row_num)) {
+ throw Exception(ErrorCode::INVALID_ARGUMENT,
+ "Malformed nullable column: inconsistent row count
{}, saved count {}",
+ row_num, saved_num);
+ }
+ }
+
size_t real_have_saved_num = 0;
buf = deserialize_const_flag_and_row_num(buf, column,
&real_have_saved_num);
- auto mem_size = real_have_saved_num * sizeof(bool);
+ if (end != nullptr) {
+ size_t remaining = static_cast<size_t>(end - buf);
+ size_t null_map_size = real_have_saved_num * sizeof(bool);
+ if (null_map_size <= SERIALIZED_MEM_SIZE_LIMIT) {
+ if (null_map_size > remaining) {
+ throw Exception(ErrorCode::INVALID_ARGUMENT,
+ "Malformed nullable column: truncated null
map");
+ }
+ } else {
+ if (remaining < sizeof(size_t)) {
+ throw Exception(ErrorCode::INVALID_ARGUMENT,
+ "Malformed nullable column: missing null map
length");
+ }
+ size_t encode_size = unaligned_load<size_t>(buf);
+ size_t payload_remaining = remaining - sizeof(size_t);
+ if (encode_size > payload_remaining ||
Review Comment:
Updated in `120abf6e41f`. The bounded nullable path now validates the
control-byte count and the sum of control-selected data bytes against
`encode_size` before decoding, then verifies the decoder consumed the same
size. Added accepted and rejected compressed null-map cases at 257 rows.
##########
be/src/core/data_type/data_type_hll.cpp:
##########
@@ -85,8 +128,21 @@ const char* DataTypeHLL::deserialize(const char* buf,
MutableColumnPtr* column,
buf += sizeof(size_t) * real_have_saved_num;
for (int i = 0; i < real_have_saved_num; ++i) {
- data[i].deserialize(Slice(buf, hll_size_array[i]));
- buf += hll_size_array[i];
+ size_t hll_size = hll_size_array[i];
+ if (hll_size > HLL_COLUMN_DEFAULT_LEN) {
+ throw Exception(ErrorCode::INVALID_ARGUMENT,
+ "Malformed HLL column: value {} is too large: {}",
i, hll_size);
+ }
+ if (end != nullptr && hll_size > static_cast<size_t>(end - buf)) {
+ throw Exception(ErrorCode::INVALID_ARGUMENT,
+ "Malformed HLL column: value {} exceeds input: {}
> {}", i, hll_size,
+ end - buf);
+ }
+ if (!data[i].deserialize(Slice(buf, hll_size))) {
Review Comment:
Updated in `120abf6e41f`. The popped `BlockItem` now releases receiver and
queue memory accounting, updates the source dependency, records counters, and
invokes `call_done()` through a function-scope cleanup guard on both success
and deserialize error. Added a backpressure test that asserts the callback runs
exactly once and the memory counter returns to zero.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
