This is an automated email from the ASF dual-hosted git repository.
HuaHuaY pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 9979ec50676 GH-50913: [C++][Dataset] Replace RapidJSON with JsonWriter
(#50914)
9979ec50676 is described below
commit 9979ec506764c2c3bb515b1ab0191534c054e8f6
Author: Aaditya Srinivasan <[email protected]>
AuthorDate: Tue Aug 25 13:44:26 2026 +0530
GH-50913: [C++][Dataset] Replace RapidJSON with JsonWriter (#50914)
### Rationale for this change
This PR continues the RapidJSON → simdjson migration by replacing the
remaining RapidJSON usage in the Dataset JSON test with Arrow's existing
`JsonWriter`.
### Changes
- Replace `rapidjson::Writer` and `rapidjson::OStreamWrapper` with
`arrow::json::JsonWriter`.
- Replace RapidJSON-specific `SizeType` usage with the `JsonWriter`
interface.
- Preserve the existing newline-delimited JSON test fixture generation.
- Remove the unused RapidJSON dependency from the Dataset test in CMake and
Meson.
- Preserve the existing Dataset JSON test coverage and behavior.
Fixes: #50913
* GitHub Issue: #50913
Authored-by: Aaditya Srinivasan <[email protected]>
Signed-off-by: Zehua Zou <[email protected]>
---
cpp/src/arrow/dataset/CMakeLists.txt | 2 +-
cpp/src/arrow/dataset/file_json_test.cc | 52 ++++++++++++++-------------------
cpp/src/arrow/dataset/meson.build | 4 +--
3 files changed, 25 insertions(+), 33 deletions(-)
diff --git a/cpp/src/arrow/dataset/CMakeLists.txt
b/cpp/src/arrow/dataset/CMakeLists.txt
index fa6875527db..321b74331e3 100644
--- a/cpp/src/arrow/dataset/CMakeLists.txt
+++ b/cpp/src/arrow/dataset/CMakeLists.txt
@@ -187,7 +187,7 @@ endif()
if(ARROW_JSON)
add_arrow_dataset_test(file_json_test EXTRA_LINK_LIBS
${ARROW_DATASET_TEST_LINK_LIBS}
- RapidJSON)
+ arrow::simdjson)
endif()
if(ARROW_ORC)
diff --git a/cpp/src/arrow/dataset/file_json_test.cc
b/cpp/src/arrow/dataset/file_json_test.cc
index 0a712a440cc..b96d4ffed99 100644
--- a/cpp/src/arrow/dataset/file_json_test.cc
+++ b/cpp/src/arrow/dataset/file_json_test.cc
@@ -20,23 +20,18 @@
#include "arrow/dataset/plan.h"
#include "arrow/dataset/test_util_internal.h"
#include "arrow/filesystem/mockfs.h"
+#include "arrow/json/json_writer_internal.h"
#include "arrow/json/parser.h"
-#include "arrow/json/rapidjson_defs.h"
#include "arrow/testing/gtest_util.h"
#include "arrow/testing/util.h"
#include "arrow/util/logging_internal.h"
-#include "rapidjson/ostreamwrapper.h"
-#include "rapidjson/writer.h"
-
namespace arrow {
using internal::checked_cast;
namespace dataset {
-namespace rj = arrow::rapidjson;
-
#define CASE(TYPE_CLASS) \
case TYPE_CLASS##Type::type_id: { \
const TYPE_CLASS##Type* concrete_ptr = nullptr; \
@@ -56,36 +51,32 @@ static Status VisitWriteableTypeId(Type::type id, VISITOR*
visitor) {
#undef CASE
-// There's currently no proper API for writing JSON files, which is reflected
in the JSON
-// dataset API as well. However, this ad-hoc implementation is good enough for
the shared
-// format test fixtures
struct WriteVisitor {
- static Status OK(bool ok) {
- return ok ? Status::OK()
- : Status::Invalid("Unexpected false return from JSON writer");
- }
-
template <typename T>
enable_if_physical_signed_integer<T, Status> Visit(const T*) {
const auto& scalar = checked_cast<const NumericScalar<T>&>(scalar_);
- return OK(writer_.Int64(scalar.value));
+ writer_.Int64(scalar.value);
+ return Status::OK();
}
template <typename T>
enable_if_physical_unsigned_integer<T, Status> Visit(const T*) {
const auto& scalar = checked_cast<const NumericScalar<T>&>(scalar_);
- return OK(writer_.Uint64(scalar.value));
+ writer_.Uint64(scalar.value);
+ return Status::OK();
}
template <typename T>
enable_if_physical_floating_point<T, Status> Visit(const T*) {
const auto& scalar = checked_cast<const NumericScalar<T>&>(scalar_);
- return OK(writer_.Double(scalar.value));
+ writer_.Double(scalar.value);
+ return Status::OK();
}
Status Visit(const BooleanType*) {
const auto& scalar = checked_cast<const BooleanScalar&>(scalar_);
- return OK(writer_.Bool(scalar.value));
+ writer_.Bool(scalar.value);
+ return Status::OK();
}
Status Visit(const StructType*) {
@@ -93,16 +84,15 @@ struct WriteVisitor {
const auto& type = checked_cast<const StructType&>(*scalar.type);
DCHECK_EQ(type.num_fields(), static_cast<int>(scalar.value.size()));
- RETURN_NOT_OK(OK(writer_.StartObject()));
+ writer_.StartObject();
for (int i = 0; i < type.num_fields(); ++i) {
const auto& name = type.field(i)->name();
- RETURN_NOT_OK(
- OK(writer_.Key(name.data(),
static_cast<rj::SizeType>(name.length()))));
+ writer_.Key(name);
const auto& child = *scalar.value[i];
if (!child.is_valid) {
- RETURN_NOT_OK(OK(writer_.Null()));
+ writer_.Null();
continue;
}
@@ -110,17 +100,16 @@ struct WriteVisitor {
RETURN_NOT_OK(VisitWriteableTypeId(child.type->id(), &visitor));
}
- RETURN_NOT_OK(OK(writer_.EndObject(type.num_fields())));
+ writer_.EndObject();
return Status::OK();
}
- rj::Writer<rj::OStreamWrapper>& writer_;
+ json::JsonWriter& writer_;
const Scalar& scalar_;
};
-Status WriteJson(const StructScalar& scalar, rj::OStreamWrapper* sink) {
- rj::Writer<rj::OStreamWrapper> writer(*sink);
- WriteVisitor visitor{writer, scalar};
+Status WriteJson(const StructScalar& scalar, json::JsonWriter* writer) {
+ WriteVisitor visitor{*writer, scalar};
return VisitWriteableTypeId(Type::STRUCT, &visitor);
}
@@ -131,10 +120,13 @@ class JsonFormatHelper {
static Result<std::shared_ptr<Buffer>> Write(RecordBatchReader* reader) {
ARROW_ASSIGN_OR_RAISE(auto scalars, ToScalars(reader));
std::stringstream ss;
- rj::OStreamWrapper sink(ss);
+
for (const auto& scalar : scalars) {
- RETURN_NOT_OK(WriteJson(*scalar, &sink));
- ss << "\n";
+ json::JsonWriter writer;
+ RETURN_NOT_OK(WriteJson(*scalar, &writer));
+
+ ARROW_ASSIGN_OR_RAISE(auto json, writer.GetString());
+ ss << json << "\n";
}
return Buffer::FromString(ss.str());
}
diff --git a/cpp/src/arrow/dataset/meson.build
b/cpp/src/arrow/dataset/meson.build
index 409ce1de2bc..df1aab46be9 100644
--- a/cpp/src/arrow/dataset/meson.build
+++ b/cpp/src/arrow/dataset/meson.build
@@ -142,7 +142,7 @@ if needs_json
dataset_tests += {
'file_json': {
'sources': ['file_json_test.cc'],
- 'dependencies': [rapidjson_dep],
+ 'dependencies': [simdjson_dep],
},
}
endif
@@ -172,7 +172,7 @@ foreach key, value : dataset_tests
exc = executable(
test_name,
sources: value['sources'],
- dependencies: [arrow_dataset_test_dep, val.get('dependencies', [])],
+ dependencies: [arrow_dataset_test_dep, value.get('dependencies', [])],
)
test(test_name, exc)
endforeach
