This is an automated email from the ASF dual-hosted git repository.
lingmiao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new e16873a Fix large string val allocation failure (#3724)
e16873a is described below
commit e16873a6c1bbf0cf9e97f7a7bfd60b162eea4e31
Author: EmmyMiao87 <[email protected]>
AuthorDate: Wed Jun 3 17:07:54 2020 +0800
Fix large string val allocation failure (#3724)
* Fix large string val allocation failure
Large bitmap will need use StringVal to allocate large memory, which is
large than MAX_INT.
The overflow will cause serialization failure of bitmap.
Fixed #3600
---
be/src/exprs/anyval_util.h | 2 +-
be/src/exprs/bitmap_function.cpp | 4 +++-
be/src/runtime/free_pool.hpp | 4 ++--
be/src/udf/udf.cpp | 10 +++++-----
be/src/udf/udf.h | 10 +++++-----
be/src/udf/udf_internal.h | 2 +-
6 files changed, 17 insertions(+), 15 deletions(-)
diff --git a/be/src/exprs/anyval_util.h b/be/src/exprs/anyval_util.h
index 2a71017..07aa95d 100755
--- a/be/src/exprs/anyval_util.h
+++ b/be/src/exprs/anyval_util.h
@@ -273,7 +273,7 @@ public:
if (type.type == FunctionContext::TYPE_VARCHAR
|| type.type == FunctionContext::TYPE_CHAR) {
DCHECK(type.len >= 0);
- val->len = std::min(val->len, type.len);
+ val->len = std::min(val->len, (int64_t)type.len);
}
}
diff --git a/be/src/exprs/bitmap_function.cpp b/be/src/exprs/bitmap_function.cpp
index 0d9bf25..6f0f02b 100644
--- a/be/src/exprs/bitmap_function.cpp
+++ b/be/src/exprs/bitmap_function.cpp
@@ -488,7 +488,9 @@ StringVal
BitmapFunctions::bitmap_from_string(FunctionContext* ctx, const String
}
std::vector<uint64_t> bits;
- if (!SplitStringAndParse({(const char*)input.ptr, input.len}, ",",
&safe_strtou64, &bits)) {
+ // The contructor of `stringpiece` only support int type.
+ if ((input.len > INT32_MAX)
+ || !SplitStringAndParse({(const char*)input.ptr, (int)input.len}, ",",
&safe_strtou64, &bits)) {
return StringVal::null();
}
diff --git a/be/src/runtime/free_pool.hpp b/be/src/runtime/free_pool.hpp
index e6efc58..0f4aad3 100755
--- a/be/src/runtime/free_pool.hpp
+++ b/be/src/runtime/free_pool.hpp
@@ -51,7 +51,7 @@ public:
virtual ~FreePool() {}
// Allocates a buffer of size.
- uint8_t* allocate(int size) {
+ uint8_t* allocate(int64_t size) {
// This is the typical malloc behavior. NULL is reserved for failures.
if (size == 0) {
return reinterpret_cast<uint8_t*>(0x1);
@@ -98,7 +98,7 @@ public:
// Returns an allocation that is at least 'size'. If the current allocation
// backing 'ptr' is big enough, 'ptr' is returned. Otherwise a new one is
// made and the contents of ptr are copied into it.
- uint8_t* reallocate(uint8_t* ptr, int size) {
+ uint8_t* reallocate(uint8_t* ptr, int64_t size) {
if (ptr == NULL || reinterpret_cast<int64_t>(ptr) == 0x1) {
return allocate(size);
}
diff --git a/be/src/udf/udf.cpp b/be/src/udf/udf.cpp
index 3270d72..e842dca 100755
--- a/be/src/udf/udf.cpp
+++ b/be/src/udf/udf.cpp
@@ -122,7 +122,7 @@ void FunctionContextImpl::close() {
_closed = true;
}
-uint8_t* FunctionContextImpl::allocate_local(int byte_size) {
+uint8_t* FunctionContextImpl::allocate_local(int64_t byte_size) {
uint8_t* buffer = _pool->allocate(byte_size);
_local_allocations.push_back(buffer);
return buffer;
@@ -365,12 +365,12 @@ bool FunctionContext::add_warning(const char*
warning_msg) {
}
}
-StringVal::StringVal(FunctionContext* context, int len) :
- len(len),
+StringVal::StringVal(FunctionContext* context, int64_t len) :
+ len(len),
ptr(context->impl()->allocate_local(len)) {
}
-bool StringVal::resize(FunctionContext* ctx, int new_len) {
+bool StringVal::resize(FunctionContext* ctx, int64_t new_len) {
if (new_len <= len) {
len = new_len;
return true;
@@ -398,7 +398,7 @@ StringVal StringVal::copy_from(FunctionContext* ctx, const
uint8_t* buf, size_t
return result;
}
-StringVal StringVal::create_temp_string_val(FunctionContext* ctx, int len) {
+StringVal StringVal::create_temp_string_val(FunctionContext* ctx, int64_t len)
{
ctx->impl()->string_result().resize(len);
return StringVal((uint8_t*)ctx->impl()->string_result().c_str(), len);
}
diff --git a/be/src/udf/udf.h b/be/src/udf/udf.h
index 80eb07e..9013d20 100755
--- a/be/src/udf/udf.h
+++ b/be/src/udf/udf.h
@@ -600,7 +600,7 @@ struct DateTimeVal : public AnyVal {
struct StringVal : public AnyVal {
static const int MAX_LENGTH = (1 << 30);
- int len;
+ int64_t len;
uint8_t* ptr;
// Construct a StringVal from ptr/len. Note: this does not make a copy of
ptr
@@ -609,7 +609,7 @@ struct StringVal : public AnyVal {
// Construct a StringVal from ptr/len. Note: this does not make a copy of
ptr
// so the buffer must exist as long as this StringVal does.
- StringVal(uint8_t* ptr, int len) : len(len), ptr(ptr) {}
+ StringVal(uint8_t* ptr, int64_t len) : len(len), ptr(ptr) {}
// Construct a StringVal from NULL-terminated c-string. Note: this does
not make a
// copy of ptr so the underlying string must exist as long as this
StringVal does.
@@ -624,12 +624,12 @@ struct StringVal : public AnyVal {
// Creates a StringVal, allocating a new buffer with 'len'. This should
// be used to return StringVal objects in UDF/UDAs that need to allocate
new
// string memory.
- StringVal(FunctionContext* context, int len);
+ StringVal(FunctionContext* context, int64_t len);
// Creates a StringVal, which memory is avaliable when this funciont
context is used next time
- static StringVal create_temp_string_val(FunctionContext* ctx, int len);
+ static StringVal create_temp_string_val(FunctionContext* ctx, int64_t len);
- bool resize(FunctionContext* context, int len);
+ bool resize(FunctionContext* context, int64_t len);
bool operator==(const StringVal& other) const {
if (is_null != other.is_null) {
diff --git a/be/src/udf/udf_internal.h b/be/src/udf/udf_internal.h
index 4ee1767..2c7a78d 100755
--- a/be/src/udf/udf_internal.h
+++ b/be/src/udf/udf_internal.h
@@ -108,7 +108,7 @@ public:
// This is used where the lifetime of the allocation is clear.
// For UDFs, the allocations can be freed at the row level.
// TODO: free them at the batch level and save some copies?
- uint8_t* allocate_local(int byte_size);
+ uint8_t* allocate_local(int64_t byte_size);
// Frees all allocations returned by AllocateLocal().
void free_local_allocations();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
