This is an automated email from the ASF dual-hosted git repository.
eldenmoon pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new b4d25a8ff5f [Fix](JsonReader) Return correct status when parse failed
(#39206) (#39269)
b4d25a8ff5f is described below
commit b4d25a8ff5f49f42df071e1be8c668270b6236b2
Author: lihangyu <[email protected]>
AuthorDate: Wed Aug 14 09:41:34 2024 +0800
[Fix](JsonReader) Return correct status when parse failed (#39206) (#39269)
#39206
---
be/src/exprs/json_functions.cpp | 9 ++++++--
.../test_json_extract_path_invalid_type.json | 13 +++++++++++
.../data/load_p0/stream_load/test_json_load.out | 4 ++++
.../load_p0/stream_load/test_json_load.groovy | 27 ++++++++++++++++++++++
4 files changed, 51 insertions(+), 2 deletions(-)
diff --git a/be/src/exprs/json_functions.cpp b/be/src/exprs/json_functions.cpp
index e83e25f7759..da5dbd82510 100644
--- a/be/src/exprs/json_functions.cpp
+++ b/be/src/exprs/json_functions.cpp
@@ -22,6 +22,7 @@
#include <rapidjson/encodings.h>
#include <rapidjson/rapidjson.h>
#include <re2/re2.h>
+#include <simdjson/error.h>
#include <simdjson/simdjson.h> // IWYU pragma: keep
#include <stdlib.h>
@@ -258,13 +259,17 @@ Status
JsonFunctions::extract_from_object(simdjson::ondemand::object& obj,
const std::vector<JsonPath>&
jsonpath,
simdjson::ondemand::value* value)
noexcept {
// Return DataQualityError when it's a malformed json.
-// Otherwise the path was not found, due to array out of bound or not exist
+// Otherwise the path was not found, due to
+// 1. array out of bound
+// 2. not exist such field in object
+// 3. the input type is not object but could be null or other types and lead
to simdjson::INCORRECT_TYPE
#define HANDLE_SIMDJSON_ERROR(err, msg)
\
do {
\
const simdjson::error_code& _err = err;
\
const std::string& _msg = msg;
\
if (UNLIKELY(_err)) {
\
- if (_err == simdjson::NO_SUCH_FIELD || _err ==
simdjson::INDEX_OUT_OF_BOUNDS) { \
+ if (_err == simdjson::NO_SUCH_FIELD || _err ==
simdjson::INDEX_OUT_OF_BOUNDS || \
+ _err == simdjson::INCORRECT_TYPE) {
\
return Status::NotFound<false>(
\
fmt::format("Not found target filed, err: {}, msg:
{}", \
simdjson::error_message(_err), _msg));
\
diff --git
a/regression-test/data/load_p0/stream_load/test_json_extract_path_invalid_type.json
b/regression-test/data/load_p0/stream_load/test_json_extract_path_invalid_type.json
new file mode 100644
index 00000000000..945b4143022
--- /dev/null
+++
b/regression-test/data/load_p0/stream_load/test_json_extract_path_invalid_type.json
@@ -0,0 +1,13 @@
+[
+ {
+ "id": 789,
+ "city": {
+ "name": "beijing",
+ "region": "haidian"
+ }
+ },
+ {
+ "id": 1111,
+ "city": null
+ }
+]
\ No newline at end of file
diff --git a/regression-test/data/load_p0/stream_load/test_json_load.out
b/regression-test/data/load_p0/stream_load/test_json_load.out
index 861cc4a1664..706d3cc3645 100644
--- a/regression-test/data/load_p0/stream_load/test_json_load.out
+++ b/regression-test/data/load_p0/stream_load/test_json_load.out
@@ -247,3 +247,7 @@ android \N \N \N \N \N
-- !select29 --
10 \N
+
+-- !select31 --
+789 beijing haidian
+1111 \N \N
diff --git a/regression-test/suites/load_p0/stream_load/test_json_load.groovy
b/regression-test/suites/load_p0/stream_load/test_json_load.groovy
index 4432ad6cc0c..51bccdddb57 100644
--- a/regression-test/suites/load_p0/stream_load/test_json_load.groovy
+++ b/regression-test/suites/load_p0/stream_load/test_json_load.groovy
@@ -759,4 +759,31 @@ suite("test_json_load", "p0") {
} finally {
try_sql("DROP TABLE IF EXISTS ${testTable}")
}
+
+ // test extract json path with invalid type(none object types like null)
+ try {
+ sql "DROP TABLE IF EXISTS ${testTable}"
+ sql """
+ CREATE TABLE ${testTable} (
+ `id` int NOT NULL,
+ `name` varchar(24) NULL,
+ `region` varchar(30) NULL
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`id`)
+ COMMENT ''
+ DISTRIBUTED BY RANDOM BUCKETS AUTO
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ );
+ """
+
+ load_json_data.call("${testTable}", "${testTable}_case31", 'true',
'false', 'json', '', '[\"$.id\", \"$.city.name\", \"$.city.region\"]',
+ '', '', '',
'test_json_extract_path_invalid_type.json', false, 2)
+
+ sql "sync"
+ qt_select31 "select * from ${testTable} order by id"
+
+ } finally {
+ // try_sql("DROP TABLE IF EXISTS ${testTable}")
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
