This is an automated email from the ASF dual-hosted git repository.
Mryange pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 1bfffb63543 [improvement](json) Lazily construct JSON error messages
(#66363)
1bfffb63543 is described below
commit 1bfffb63543a90699944673d50031066b279644a
Author: York Cao <[email protected]>
AuthorDate: Mon Aug 3 15:22:46 2026 +0800
[improvement](json) Lazily construct JSON error messages (#66363)
### What problem does this PR solve?
Issue Number: close #66362
Related PR: None
Problem Summary:
JSON path extraction eagerly formats an error message at every
`HANDLE_SIMDJSON_ERROR` call before checking whether simdjson reported
an error. Successful lookups therefore pay repeated formatting and
allocation costs even though the message is unused.
This change constructs the message only inside the unlikely error
branch. It keeps owned string storage on that branch so formatted
temporaries remain safe. Focused coverage verifies missing fields,
out-of-bounds array indexes, and incorrect intermediate types, and a
benchmark covers successful and missing-field lookups.
In a local same-session RELEASE A/B benchmark over 4,096 rows, the
valid-field median improved from 689,024 ns to 364,504 ns (about 1.89x
throughput); the missing-field path remained approximately unchanged.
### Release note
None
### Check List (For Author)
- Test
- [ ] Regression test
- [x] Unit Test
- [x] Manual test: `./build.sh --be`, `./build.sh --benchmark`, and the
A/B benchmark described above
- [ ] No need to test or manual test.
- Behavior changed:
- [x] No.
- [ ] Yes.
- Does this need documentation?
- [x] No.
- [ ] Yes.
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
---
be/benchmark/benchmark_json_extract.hpp | 76 +++++++++++++++++++++++++++++++++
be/benchmark/benchmark_main.cpp | 1 +
be/src/exprs/json_functions.cpp | 3 +-
be/test/exprs/json_function_test.cpp | 25 +++++++++++
4 files changed, 104 insertions(+), 1 deletion(-)
diff --git a/be/benchmark/benchmark_json_extract.hpp
b/be/benchmark/benchmark_json_extract.hpp
new file mode 100644
index 00000000000..87c6dce4e97
--- /dev/null
+++ b/be/benchmark/benchmark_json_extract.hpp
@@ -0,0 +1,76 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <benchmark/benchmark.h>
+#include <simdjson.h>
+
+#include <string>
+#include <vector>
+
+#include "common/status.h"
+#include "exprs/json_functions.h"
+
+namespace doris {
+namespace {
+
+constexpr int JSON_EXTRACT_ROWS = 4096;
+
+std::vector<std::string> make_json_rows() {
+ std::vector<std::string> rows;
+ rows.reserve(JSON_EXTRACT_ROWS);
+ for (int i = 0; i < JSON_EXTRACT_ROWS; ++i) {
+ rows.emplace_back(
+
R"({"skuBaselineFlag":"Y","grossProfitBaseline":123.45,"handPriceBaseline":678.9,)"
+ R"("target":"abc","grossRateBaseline":0.1234})");
+ }
+ return rows;
+}
+
+void run_json_extract(benchmark::State& state, const std::string& path) {
+ const std::vector<std::string> rows = make_json_rows();
+ std::vector<JsonPath> parsed_paths;
+ JsonFunctions::parse_json_paths(path, &parsed_paths);
+ simdjson::ondemand::parser parser;
+
+ for (auto _ : state) {
+ for (const auto& row : rows) {
+ simdjson::padded_string json(row.data(), row.size());
+ auto document = parser.iterate(json);
+ simdjson::ondemand::object object = document.get_object();
+ simdjson::ondemand::value value;
+ Status status = JsonFunctions::extract_from_object(object,
parsed_paths, &value);
+ benchmark::DoNotOptimize(status);
+ benchmark::DoNotOptimize(value);
+ }
+ }
+ state.SetItemsProcessed(state.iterations() *
static_cast<int64_t>(rows.size()));
+}
+
+void JsonExtractFromObjectValidField(benchmark::State& state) {
+ run_json_extract(state, "$.target");
+}
+BENCHMARK(JsonExtractFromObjectValidField);
+
+void JsonExtractFromObjectMissingField(benchmark::State& state) {
+ run_json_extract(state, "$.does_not_exist");
+}
+BENCHMARK(JsonExtractFromObjectMissingField);
+
+} // namespace
+} // namespace doris
diff --git a/be/benchmark/benchmark_main.cpp b/be/benchmark/benchmark_main.cpp
index 7c64fa2729c..5eee068d752 100644
--- a/be/benchmark/benchmark_main.cpp
+++ b/be/benchmark/benchmark_main.cpp
@@ -34,6 +34,7 @@
#include "benchmark_fmod.hpp"
#include "benchmark_hll_merge.hpp"
#include "benchmark_hybrid_set.hpp"
+#include "benchmark_json_extract.hpp"
#include "benchmark_pdep_unpack.hpp"
#include "benchmark_string.hpp"
#include "benchmark_string_replace.hpp"
diff --git a/be/src/exprs/json_functions.cpp b/be/src/exprs/json_functions.cpp
index 76d2047ac43..228ab8c5e03 100644
--- a/be/src/exprs/json_functions.cpp
+++ b/be/src/exprs/json_functions.cpp
@@ -263,11 +263,12 @@ Status
JsonFunctions::extract_from_object(simdjson::ondemand::object& obj,
// 1. array out of bound
// 2. not exist such field in object
// 3. the input type is not object but could be null or other types and lead
to simdjson::INCORRECT_TYPE
+// Construct msg only on error because call sites use fmt::format and most
lookups succeed.
#define HANDLE_SIMDJSON_ERROR(err, msg)
\
do {
\
const simdjson::error_code& _err = err;
\
- const std::string& _msg = msg;
\
if (UNLIKELY(_err)) {
\
+ const std::string _msg = msg;
\
if (_err == simdjson::NO_SUCH_FIELD || _err ==
simdjson::INDEX_OUT_OF_BOUNDS || \
_err == simdjson::INCORRECT_TYPE) {
\
return Status::NotFound<false>(
\
diff --git a/be/test/exprs/json_function_test.cpp
b/be/test/exprs/json_function_test.cpp
index fcb1c898dea..d2d9b1b8fb3 100644
--- a/be/test/exprs/json_function_test.cpp
+++ b/be/test/exprs/json_function_test.cpp
@@ -23,6 +23,7 @@
#include <rapidjson/writer.h>
#include <string>
+#include <string_view>
#include "exprs/json_functions.h"
#include "gtest/gtest_pred_impl.h"
@@ -127,4 +128,28 @@ TEST_F(JsonFunctionTest, json_path_test) {
}
}
+TEST_F(JsonFunctionTest, extract_from_object_not_found_errors) {
+ const auto expect_not_found = [](std::string_view json, const std::string&
path,
+ std::string_view expected_message) {
+ simdjson::ondemand::parser parser;
+ simdjson::padded_string padded_json(json.data(), json.size());
+ auto document = parser.iterate(padded_json);
+ simdjson::ondemand::object object;
+ ASSERT_EQ(document.get_object().get(object), simdjson::SUCCESS);
+
+ std::vector<JsonPath> parsed_paths;
+ JsonFunctions::parse_json_paths(path, &parsed_paths);
+ simdjson::ondemand::value value;
+ const Status status = JsonFunctions::extract_from_object(object,
parsed_paths, &value);
+
+ ASSERT_TRUE(status.is<ErrorCode::NOT_FOUND>()) << status;
+ EXPECT_NE(status.to_string().find(expected_message),
std::string::npos) << status;
+ };
+
+ expect_not_found(R"({"k1":{"k2":"v2"}})", "$.k1.k3", "unable to find
field: k3");
+ expect_not_found(R"({"k1":{"k2":[1,2,3]}})", "$.k1.k2[5]",
+ "failed to access array field: k2, index: 5");
+ expect_not_found(R"({"k1":{"k2":"v2"}})", "$.k1.k2.k3", "unable to find
field: k3");
+}
+
} // namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
