This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new e76f712 [Bug] Load data is error in json load
e76f712 is described below
commit e76f712bb353dc8f9b5c1f893b567048abda8ded
Author: worker24h <[email protected]>
AuthorDate: Thu May 28 17:28:33 2020 +0800
[Bug] Load data is error in json load
---
be/src/exec/json_scanner.cpp | 12 ++++--------
be/src/exprs/json_functions.cpp | 9 ++++++++-
be/src/exprs/json_functions.h | 3 ++-
be/test/exprs/json_function_test.cpp | 10 +++++-----
4 files changed, 19 insertions(+), 15 deletions(-)
diff --git a/be/src/exec/json_scanner.cpp b/be/src/exec/json_scanner.cpp
index 1a36591..89fdeef 100644
--- a/be/src/exec/json_scanner.cpp
+++ b/be/src/exec/json_scanner.cpp
@@ -267,15 +267,11 @@ size_t JsonReader::get_data_by_jsonpath(const
std::vector<SlotDescriptor*>& slot
}
// if jsonValues is null, because not match in jsondata.
- rapidjson::Value* json_values =
JsonFunctions::get_json_object_from_parsed_json(path.GetString(), &_json_doc,
_json_doc.GetAllocator());
+ rapidjson::Value* json_values =
JsonFunctions::get_json_array_from_parsed_json(path.GetString(), &_json_doc,
_json_doc.GetAllocator());
if (json_values == nullptr) {
return -1;
}
- if (json_values->IsArray()) {
- max_lines = std::max(max_lines, (size_t)json_values->Size());
- } else {
- max_lines = std::max(max_lines, (size_t)1);
- }
+ max_lines = std::max(max_lines, (size_t)json_values->Size());
_jmap.emplace(slot_descs[i]->col_name(), json_values);
}
@@ -376,7 +372,7 @@ Status JsonReader::set_tuple_value(rapidjson::Value&
objectValue, Tuple* tuple,
/**
* handle input a simple json
* For example:
- * case 1. {"RECORDS": [{"colunm1":"value1", "colunm2":10},
{"colunm1":"value2", "colunm2":30}]}
+ * case 1. [{"colunm1":"value1", "colunm2":10}, {"colunm1":"value2",
"colunm2":30}]
* case 2. {"colunm1":"value1", "colunm2":10}
*/
Status JsonReader::handle_simple_json(Tuple* tuple, const
std::vector<SlotDescriptor*>& slot_descs, MemPool* tuple_pool, bool* eof) {
@@ -497,7 +493,7 @@ Status JsonReader::handle_flat_array_complex_json(Tuple*
tuple, const std::vecto
}
// if jsonValues is null, because not match in jsondata.
- rapidjson::Value* json_values =
JsonFunctions::get_json_object_from_parsed_json(path.GetString(), &objectValue,
_json_doc.GetAllocator());
+ rapidjson::Value* json_values =
JsonFunctions::get_json_array_from_parsed_json(path.GetString(), &objectValue,
_json_doc.GetAllocator());
if (json_values == nullptr) {
if (slot_descs[i]->is_nullable()) {
tuple->set_null(slot_descs[i]->null_indicator_offset());
diff --git a/be/src/exprs/json_functions.cpp b/be/src/exprs/json_functions.cpp
index 90842fb..e7252e4 100644
--- a/be/src/exprs/json_functions.cpp
+++ b/be/src/exprs/json_functions.cpp
@@ -263,7 +263,7 @@ rapidjson::Value* JsonFunctions::get_json_object(
}
-rapidjson::Value* JsonFunctions::get_json_object_from_parsed_json (
+rapidjson::Value* JsonFunctions::get_json_array_from_parsed_json (
const std::string& path_string,
rapidjson::Value* document,
rapidjson::Document::AllocatorType& mem_allocator) {
@@ -293,6 +293,13 @@ rapidjson::Value*
JsonFunctions::get_json_object_from_parsed_json (
rapidjson::Value* root = match_value(parsed_paths, document,
mem_allocator, true);
if (root == document) {// not found
return nullptr;
+ } else if (!root->IsArray()) {
+ rapidjson::Value* array_obj = nullptr;
+ array_obj = static_cast<rapidjson::Value*>(
+ mem_allocator.Malloc(sizeof(rapidjson::Value)));
+ array_obj->SetArray();
+ array_obj->PushBack(*root, mem_allocator);
+ return array_obj;
}
return root;
}
diff --git a/be/src/exprs/json_functions.h b/be/src/exprs/json_functions.h
index 24513b2..2d7a53c 100644
--- a/be/src/exprs/json_functions.h
+++ b/be/src/exprs/json_functions.h
@@ -72,8 +72,9 @@ public:
/**
* The `document` parameter must be has parsed.
+ * return Value Is Array object
*/
- static rapidjson::Value* get_json_object_from_parsed_json(
+ static rapidjson::Value* get_json_array_from_parsed_json(
const std::string& path_string,
rapidjson::Value* document,
rapidjson::Document::AllocatorType& mem_allocator);
diff --git a/be/test/exprs/json_function_test.cpp
b/be/test/exprs/json_function_test.cpp
index 6595b22..95598df 100644
--- a/be/test/exprs/json_function_test.cpp
+++ b/be/test/exprs/json_function_test.cpp
@@ -194,13 +194,13 @@ TEST_F(JsonFunctionTest, json_path1)
ASSERT_TRUE(false);
}
rapidjson::Value* res3;
- res3 = JsonFunctions::get_json_object_from_parsed_json("$.[*].keyname.ip",
&jsonDoc, jsonDoc.GetAllocator());
+ res3 = JsonFunctions::get_json_array_from_parsed_json("$.[*].keyname.ip",
&jsonDoc, jsonDoc.GetAllocator());
ASSERT_TRUE(res3->IsArray());
for (int i = 0; i < res3->Size(); i++) {
std::cout<< (*res3)[i].GetString() << std::endl;
}
- res3 = JsonFunctions::get_json_object_from_parsed_json("$.[*].k1",
&jsonDoc, jsonDoc.GetAllocator());
+ res3 = JsonFunctions::get_json_array_from_parsed_json("$.[*].k1",
&jsonDoc, jsonDoc.GetAllocator());
ASSERT_TRUE(res3->IsArray());
for (int i = 0; i < res3->Size(); i++) {
std::cout<< (*res3)[i].GetString() << std::endl;
@@ -216,7 +216,7 @@ TEST_F(JsonFunctionTest, json_path_get_nullobject)
ASSERT_TRUE(false);
}
- rapidjson::Value* res3 =
JsonFunctions::get_json_object_from_parsed_json("$.[*].b", &jsonDoc,
jsonDoc.GetAllocator());
+ rapidjson::Value* res3 =
JsonFunctions::get_json_array_from_parsed_json("$.[*].b", &jsonDoc,
jsonDoc.GetAllocator());
ASSERT_TRUE(res3->IsArray());
ASSERT_EQ(res3->Size(), 3);
for (int i = 0; i < res3->Size(); i++) {
@@ -239,7 +239,7 @@ TEST_F(JsonFunctionTest, json_path_test)
ASSERT_TRUE(false);
}
- rapidjson::Value* res3 =
JsonFunctions::get_json_object_from_parsed_json("$.[*].a", &jsonDoc,
jsonDoc.GetAllocator());
+ rapidjson::Value* res3 =
JsonFunctions::get_json_array_from_parsed_json("$.[*].a", &jsonDoc,
jsonDoc.GetAllocator());
ASSERT_TRUE(res3->IsArray());
ASSERT_EQ(res3->Size(), 2);
for (int i = 0; i < res3->Size(); i++) {
@@ -258,7 +258,7 @@ TEST_F(JsonFunctionTest, json_path_test)
ASSERT_TRUE(false);
}
- rapidjson::Value* res3 =
JsonFunctions::get_json_object_from_parsed_json("$.a", &jsonDoc,
jsonDoc.GetAllocator());
+ rapidjson::Value* res3 =
JsonFunctions::get_json_array_from_parsed_json("$.a", &jsonDoc,
jsonDoc.GetAllocator());
ASSERT_TRUE(res3->IsArray());
ASSERT_EQ(res3->Size(), 2);
for (int i = 0; i < res3->Size(); i++) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
