pitrou commented on code in PR #50874:
URL: https://github.com/apache/arrow/pull/50874#discussion_r3842087296
##########
cpp/src/arrow/extension/tensor_extension_array_test.cc:
##########
@@ -212,7 +212,7 @@ TEST_F(TestFixedShapeTensorType,
MetadataSerializationRoundtrip) {
CheckDeserializationRaises(ext_type_, storage_type,
R"({"dim_names":["x","y"]})",
"Invalid serialized JSON data");
CheckDeserializationRaises(ext_type_, storage_type, R"({"shape":(3,4)})",
- "Invalid serialized JSON data");
+ "shape must be an array, got unknown");
Review Comment:
That's a weird error message for a parse error, can we improve that?
##########
cpp/src/arrow/extension/fixed_shape_tensor.cc:
##########
@@ -116,60 +115,179 @@ Result<std::shared_ptr<DataType>>
FixedShapeTensorType::Deserialize(
return Status::Invalid("Expected FixedSizeList storage type, got ",
storage_type->ToString());
}
+
auto fsl_type =
internal::checked_pointer_cast<FixedSizeListType>(storage_type);
auto value_type = fsl_type->value_type();
- rj::Document document;
- if (document.Parse(serialized_data.data(),
serialized_data.length()).HasParseError() ||
- !document.IsObject() || !document.HasMember("shape") ||
- !document["shape"].IsArray()) {
- return Status::Invalid("Invalid serialized JSON data: ", serialized_data);
- }
- std::vector<int64_t> shape;
- for (const auto& x : document["shape"].GetArray()) {
- if (!x.IsInt64()) {
- return Status::Invalid("shape must contain integers, got ",
- internal::JsonTypeName(x));
- }
- shape.emplace_back(x.GetInt64());
- }
+ simdjson::padded_string padded_json(serialized_data);
+ simdjson::ondemand::parser parser;
+
+ ARROW_ASSIGN_OR_RAISE(auto document,
+
internal::ResolveSimdjsonResult(parser.iterate(padded_json),
+ "Invalid serialized
JSON data"));
+ ARROW_ASSIGN_OR_RAISE(auto object,
+ internal::ResolveSimdjsonResult(document.get_object(),
+ "Invalid serialized
JSON data"));
+
+ std::vector<int64_t> shape;
std::vector<int64_t> permutation;
- if (document.HasMember("permutation")) {
- const auto& json_permutation = document["permutation"];
- if (!json_permutation.IsArray()) {
- return Status::Invalid("permutation must be an array, got ",
- internal::JsonTypeName(json_permutation));
- }
- for (const auto& x : json_permutation.GetArray()) {
- if (!x.IsInt64()) {
- return Status::Invalid("permutation must contain integers, got ",
- internal::JsonTypeName(x));
+ std::vector<std::string> dim_names;
+
+ bool has_shape = false;
+
+ for (auto field_result : object) {
+ ARROW_ASSIGN_OR_RAISE(auto field, internal::ResolveSimdjsonResult(
+ field_result, "Failed to iterate
JSON object"));
Review Comment:
We're seeing this kind of logic a lot, is there a way to build up our
simdjson utilities to reduce the boilerplate here?
##########
cpp/src/arrow/extension/fixed_shape_tensor.cc:
##########
@@ -116,60 +115,179 @@ Result<std::shared_ptr<DataType>>
FixedShapeTensorType::Deserialize(
return Status::Invalid("Expected FixedSizeList storage type, got ",
storage_type->ToString());
}
+
auto fsl_type =
internal::checked_pointer_cast<FixedSizeListType>(storage_type);
auto value_type = fsl_type->value_type();
- rj::Document document;
- if (document.Parse(serialized_data.data(),
serialized_data.length()).HasParseError() ||
- !document.IsObject() || !document.HasMember("shape") ||
- !document["shape"].IsArray()) {
- return Status::Invalid("Invalid serialized JSON data: ", serialized_data);
- }
- std::vector<int64_t> shape;
- for (const auto& x : document["shape"].GetArray()) {
- if (!x.IsInt64()) {
- return Status::Invalid("shape must contain integers, got ",
- internal::JsonTypeName(x));
- }
- shape.emplace_back(x.GetInt64());
- }
+ simdjson::padded_string padded_json(serialized_data);
+ simdjson::ondemand::parser parser;
Review Comment:
Would using the DOM API make things easier? This path is not
performance-critical.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
