This is an automated email from the ASF dual-hosted git repository.

yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/master by this push:
     new 6bfb3c8d8b0 [fix](mem) heap-buffer-overflow for function convert_to 
(#46405)
6bfb3c8d8b0 is described below

commit 6bfb3c8d8b07cf31d6de335d1f630ba65e53254e
Author: camby <[email protected]>
AuthorDate: Mon Jan 6 21:38:55 2025 +0800

    [fix](mem) heap-buffer-overflow for function convert_to (#46405)
    
    ### What problem does this PR solve?
    
    Reproduce SQL with ASAN version:
    `select convert('装装装装装' using gbk);`
    
    Then be crashed:
    ```
    =================================================================
    ==1830466==ERROR: AddressSanitizer: heap-buffer-overflow on address 
0x606002aeec20 at pc 0x560826fb3e66 bp 0x7fc3816a5890 sp 0x7fc3816a5058
    
    WRITE of size 10 at 0x606002aeec20 thread T711 (brpc_light)
    ```
    
    But if we use release version, we found the result is not correct, and
    the memory maybe already corrupted:
    ```
    > select convert('装装装装装' using gbk);
    +---------------------------------------------------------+
    | convert_to('装装装装装', 'gbk')                         |
    +---------------------------------------------------------+
    | ~zhuangdang~zhuangdang~zhuangdang~zhuangdang~zhu        |
    +---------------------------------------------------------+
    1 row in set (0.04 sec)
    ```
    
    The correct answer should be:
    ```
    > select convert('装装装装装' using gbk);
    +--------------------------------------+
    | convert_to('装装装装装', 'gbk')      |
    +--------------------------------------+
    | ~zhuang~zhuang~zhuang~zhuang~zhuang  |
    +--------------------------------------+
    1 row in set (0.06 sec)
    ```
---
 be/src/vec/functions/function_string.h                |  13 +++++++++----
 .../data/nereids_function_p0/scalar_function/C.out    |   3 +++
 .../string_functions/test_string_function.out         | Bin 4838 -> 4892 bytes
 .../nereids_function_p0/scalar_function/C.groovy      |   2 +-
 .../string_functions/test_string_function.groovy      |   2 ++
 5 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/be/src/vec/functions/function_string.h 
b/be/src/vec/functions/function_string.h
index a729af5948a..1eb43716806 100644
--- a/be/src/vec/functions/function_string.h
+++ b/be/src/vec/functions/function_string.h
@@ -3770,9 +3770,10 @@ public:
         auto& res_offset = col_res->get_offsets();
         auto& res_chars = col_res->get_chars();
         res_offset.resize(input_rows_count);
-        // max pinyin size is 6, double of utf8 chinese word 3, add one char 
to set '~'
-        ColumnString::check_chars_length(str_chars.size() * 2 + 
input_rows_count, 0);
-        res_chars.resize(str_chars.size() * 2 + input_rows_count);
+        // max pinyin size is 6 + 1 (first '~') for utf8 chinese word 3
+        size_t pinyin_size = (str_chars.size() + 2) / 3 * 7;
+        ColumnString::check_chars_length(pinyin_size, 0);
+        res_chars.resize(pinyin_size);
 
         size_t in_len = 0, out_len = 0;
         for (int i = 0; i < input_rows_count; ++i) {
@@ -3813,7 +3814,11 @@ public:
                     }
 
                     auto end = strchr(buf, ' ');
-                    auto len = end != nullptr ? end - buf : MAX_PINYIN_LEN;
+                    // max len for pinyin is 6
+                    int len = MAX_PINYIN_LEN;
+                    if (end != nullptr && end - buf < MAX_PINYIN_LEN) {
+                        len = end - buf;
+                    }
                     // set first char '~' just make sure all english word 
lower than chinese word
                     *dest = 126;
                     memcpy(dest + 1, buf, len);
diff --git a/regression-test/data/nereids_function_p0/scalar_function/C.out 
b/regression-test/data/nereids_function_p0/scalar_function/C.out
index e36fdc7f374..924ddf5b148 100644
--- a/regression-test/data/nereids_function_p0/scalar_function/C.out
+++ b/regression-test/data/nereids_function_p0/scalar_function/C.out
@@ -1418,3 +1418,6 @@ Doris     Doris
 -- !convert --
 1      1
 
+-- !convert_1 --
+~zhuang~zhuang~zhuang~zhuang~zhuang
+
diff --git 
a/regression-test/data/query_p0/sql_functions/string_functions/test_string_function.out
 
b/regression-test/data/query_p0/sql_functions/string_functions/test_string_function.out
index cadf5039794..4af2997eda2 100644
Binary files 
a/regression-test/data/query_p0/sql_functions/string_functions/test_string_function.out
 and 
b/regression-test/data/query_p0/sql_functions/string_functions/test_string_function.out
 differ
diff --git 
a/regression-test/suites/nereids_function_p0/scalar_function/C.groovy 
b/regression-test/suites/nereids_function_p0/scalar_function/C.groovy
index bf072c9ad8c..0a14602eff9 100644
--- a/regression-test/suites/nereids_function_p0/scalar_function/C.groovy
+++ b/regression-test/suites/nereids_function_p0/scalar_function/C.groovy
@@ -197,5 +197,5 @@ suite("nereids_scalar_fn_C") {
        qt_bitmap_not_nullable "select count(kbitmap) from 
fn_test_bitmap_not_nullable"
        qt_char "select char(68, 111, 114, 105, 115), char(68, 111, 114, 105, 
115 using utf8);"
        qt_convert "select convert(1 using gbk), convert(1, string);"
-
+       qt_convert_1 "select convert('装装装装装' using gbk);"
 }
diff --git 
a/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function.groovy
 
b/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function.groovy
index 6e18fb57eeb..c1a3e6b57fe 100644
--- 
a/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function.groovy
+++ 
b/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function.groovy
@@ -163,6 +163,8 @@ suite("test_string_function", "arrow_flight_sql") {
     qt_sql "select right(\"Hello doris\", 120);"
     qt_sql "select right(\"Hello doris\", -6);"
 
+    qt_convert_1 "select convert('装装装装装' using gbk);"
+
     sql """ drop table if exists left_right_test; """
     sql """ create table left_right_test (
         id INT NULL,


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to