This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.2-lts by this push:
new fade2feac3 [bugfix](jsonb) fix jsonb parser crash on noavx2 host
(#18977) (#19454)
fade2feac3 is described below
commit fade2feac3c6df651c31d453af8ca811e7413177
Author: Kang <[email protected]>
AuthorDate: Wed May 10 11:05:11 2023 +0800
[bugfix](jsonb) fix jsonb parser crash on noavx2 host (#18977) (#19454)
support avx2 and noavx2 for jsonb parser using __AVX2__ macro.
---
be/src/runtime/jsonb_value.h | 6 +++++-
be/src/util/jsonb_document.h | 3 +++
be/src/util/jsonb_parser.h | 3 +++
be/src/util/jsonb_parser_simd.h | 2 +-
be/src/vec/functions/function_json.cpp | 5 +++++
be/src/vec/functions/function_jsonb.cpp | 10 +++++++---
6 files changed, 24 insertions(+), 5 deletions(-)
diff --git a/be/src/runtime/jsonb_value.h b/be/src/runtime/jsonb_value.h
index 1e03518d9d..3196b0ab33 100644
--- a/be/src/runtime/jsonb_value.h
+++ b/be/src/runtime/jsonb_value.h
@@ -22,7 +22,11 @@
#include "util/cpu_info.h"
#include "util/hash_util.hpp"
#include "util/jsonb_error.h"
+#ifdef __AVX2__
#include "util/jsonb_parser_simd.h"
+#else
+#include "util/jsonb_parser.h"
+#endif
#include "util/jsonb_utils.h"
#include "vec/common/string_ref.h"
@@ -38,7 +42,7 @@ struct JsonBinaryValue {
// default nullprt and size 0 for invalid or NULL value
const char* ptr = nullptr;
size_t len = 0;
- JsonbParserSIMD parser;
+ JsonbParser parser;
JsonBinaryValue() : ptr(nullptr), len(0) {}
JsonBinaryValue(char* ptr, int len) { from_json_string(const_cast<const
char*>(ptr), len); }
diff --git a/be/src/util/jsonb_document.h b/be/src/util/jsonb_document.h
index 98f98b8377..62a9d0b2da 100644
--- a/be/src/util/jsonb_document.h
+++ b/be/src/util/jsonb_document.h
@@ -74,6 +74,9 @@
// #include "util/string_parser.hpp"
+#include "common/compiler_util.h"
+#include "common/logging.h"
+
namespace doris {
#pragma pack(push, 1)
diff --git a/be/src/util/jsonb_parser.h b/be/src/util/jsonb_parser.h
index f4711f9a62..8d29b52faf 100644
--- a/be/src/util/jsonb_parser.h
+++ b/be/src/util/jsonb_parser.h
@@ -665,12 +665,15 @@ private:
case 4:
*--out = ((uc | 0x80) & 0xBF);
uc >>= 6;
+ [[fallthrough]];
case 3:
*--out = ((uc | 0x80) & 0xBF);
uc >>= 6;
+ [[fallthrough]];
case 2:
*--out = ((uc | 0x80) & 0xBF);
uc >>= 6;
+ [[fallthrough]];
case 1:
// Mask the first byte according to the standard.
*--out = (uc | firstByteMark[len - 1]);
diff --git a/be/src/util/jsonb_parser_simd.h b/be/src/util/jsonb_parser_simd.h
index 10d19a3f57..971d06d46e 100644
--- a/be/src/util/jsonb_parser_simd.h
+++ b/be/src/util/jsonb_parser_simd.h
@@ -343,7 +343,7 @@ private:
JsonbErrType err_;
};
-using JsonbParserSIMD = JsonbParserTSIMD<JsonbOutStream>;
+using JsonbParser = JsonbParserTSIMD<JsonbOutStream>;
} // namespace doris
diff --git a/be/src/vec/functions/function_json.cpp
b/be/src/vec/functions/function_json.cpp
index 4014b65610..e85e29c5a1 100644
--- a/be/src/vec/functions/function_json.cpp
+++ b/be/src/vec/functions/function_json.cpp
@@ -23,6 +23,11 @@
#include <vector>
#include "exprs/json_functions.h"
+#ifdef __AVX2__
+#include "util/jsonb_parser_simd.h"
+#else
+#include "util/jsonb_parser.h"
+#endif
#include "util/string_parser.hpp"
#include "util/string_util.h"
#include "vec/columns/column.h"
diff --git a/be/src/vec/functions/function_jsonb.cpp
b/be/src/vec/functions/function_jsonb.cpp
index 02f352fb57..ea61d5f5e0 100644
--- a/be/src/vec/functions/function_jsonb.cpp
+++ b/be/src/vec/functions/function_jsonb.cpp
@@ -18,7 +18,11 @@
#include <boost/token_functions.hpp>
#include <vector>
-// #include "util/jsonb_parser_simd.h"
+#ifdef __AVX2__
+#include "util/jsonb_parser_simd.h"
+#else
+#include "util/jsonb_parser.h"
+#endif
#include "util/string_parser.hpp"
#include "util/string_util.h"
#include "vec/columns/column.h"
@@ -48,7 +52,7 @@ enum class JsonbParseErrorMode { FAIL = 0, RETURN_NULL,
RETURN_VALUE, RETURN_INV
template <NullalbeMode nullable_mode, JsonbParseErrorMode
parse_error_handle_mode>
class FunctionJsonbParseBase : public IFunction {
private:
- JsonbParserSIMD default_value_parser;
+ JsonbParser default_value_parser;
bool has_const_default_value = false;
public:
@@ -195,7 +199,7 @@ public:
col_to->reserve(size);
// parser can be reused for performance
- JsonbParserSIMD parser;
+ JsonbParser parser;
JsonbErrType error = JsonbErrType::E_NONE;
for (size_t i = 0; i < input_rows_count; ++i) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
