This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new ba6c5fecdd3 GH-51638: [CI][C++][Parquet] Update expected JSON output
in parquet-reader-test for simdjson 5.0 (#51661)
ba6c5fecdd3 is described below
commit ba6c5fecdd3688d53e65a48700f08c58df6248c8
Author: tadeja <[email protected]>
AuthorDate: Thu Oct 1 23:19:18 2026 +0200
GH-51638: [CI][C++][Parquet] Update expected JSON output in
parquet-reader-test for simdjson 5.0 (#51661)
### Rationale for this change
See #51638 - `fractured_json` formatting changes with the new simdjson
version 5.0 (whitespace, line breaks, key order in tables),
cause `parquet-reader-test` to fail on four of its tests
`TestJSONWithLocalFile.JSONOutput...`
### What changes are included in this PR?
Check `simdjson::SIMDJSON_VERSION_MAJOR`, and for `>= 5` use the new
expected formatting (JSON content hasn't changed).
Keep existing expected output formatting for simdjson 4.x
### Are these changes tested?
Yes, CI runs pass.
### Are there any user-facing changes?
No, only test changes.
### Was AI used for this PR?
**PR code and description written by:**
- [x] Human
- [x] AI
**Reviewed before submission by:**
- [x] Human
- [ ] AI
- [ ] Not reviewed
* GitHub Issue: #51638
Authored-by: Tadeja Kadunc <[email protected]>
Signed-off-by: Sutou Kouhei <[email protected]>
---
cpp/src/parquet/reader_test.cc | 228 +++++++++++++++++++++++++++++++++++------
1 file changed, 196 insertions(+), 32 deletions(-)
diff --git a/cpp/src/parquet/reader_test.cc b/cpp/src/parquet/reader_test.cc
index e4d2fd1d590..b5487fa4689 100644
--- a/cpp/src/parquet/reader_test.cc
+++ b/cpp/src/parquet/reader_test.cc
@@ -1124,6 +1124,11 @@ class TestJSONWithLocalFile : public ::testing::Test {
};
TEST_F(TestJSONWithLocalFile, JSONOutputWithStatistics) {
+ // simdjson 5.0 changed the format of fractured_json
+ if constexpr (simdjson::SIMDJSON_VERSION_MAJOR < 5) {
+ GTEST_SKIP() << "Test requires simdjson >= 5";
+ }
+
std::string json_output = R"###({
"FileName": "nested_lists.snappy.parquet",
"Version": "1.0",
@@ -1138,15 +1143,9 @@ TEST_F(TestJSONWithLocalFile, JSONOutputWithStatistics) {
"Name": "a.list.element.list.element.list.element",
"PhysicalType": "BYTE_ARRAY",
"ConvertedType": "UTF8",
- "LogicalType": { "Type": "String" }
+ "LogicalType": {"Type": "String"}
},
- {
- "Id": "1",
- "Name": "b",
- "PhysicalType": "INT32",
- "ConvertedType": "NONE",
- "LogicalType": { "Type": "None" }
- }
+ { "Id": "1", "Name": "b", "PhysicalType": "INT32", "ConvertedType":
"NONE", "LogicalType": {"Type": "None"} }
],
"RowGroups": [
{
@@ -1191,6 +1190,11 @@ TEST_F(TestJSONWithLocalFile, JSONOutputWithStatistics) {
}
TEST_F(TestJSONWithLocalFile, JSONOutput) {
+ // simdjson 5.0 changed the format of fractured_json
+ if constexpr (simdjson::SIMDJSON_VERSION_MAJOR < 5) {
+ GTEST_SKIP() << "Test requires simdjson >= 5";
+ }
+
std::string json_output = R"###({
"FileName": "alltypes_plain.parquet",
"Version": "1.0",
@@ -1200,17 +1204,77 @@ TEST_F(TestJSONWithLocalFile, JSONOutput) {
"NumberOfRealColumns": "11",
"NumberOfColumns": "11",
"Columns": [
- { "ConvertedType": "NONE", "Id": "0" , "LogicalType": { "Type": "None"
}, "Name": "id" , "PhysicalType": "INT32" },
- { "ConvertedType": "NONE", "Id": "1" , "LogicalType": { "Type": "None"
}, "Name": "bool_col" , "PhysicalType": "BOOLEAN" },
- { "ConvertedType": "NONE", "Id": "2" , "LogicalType": { "Type": "None"
}, "Name": "tinyint_col" , "PhysicalType": "INT32" },
- { "ConvertedType": "NONE", "Id": "3" , "LogicalType": { "Type": "None"
}, "Name": "smallint_col" , "PhysicalType": "INT32" },
- { "ConvertedType": "NONE", "Id": "4" , "LogicalType": { "Type": "None"
}, "Name": "int_col" , "PhysicalType": "INT32" },
- { "ConvertedType": "NONE", "Id": "5" , "LogicalType": { "Type": "None"
}, "Name": "bigint_col" , "PhysicalType": "INT64" },
- { "ConvertedType": "NONE", "Id": "6" , "LogicalType": { "Type": "None"
}, "Name": "float_col" , "PhysicalType": "FLOAT" },
- { "ConvertedType": "NONE", "Id": "7" , "LogicalType": { "Type": "None"
}, "Name": "double_col" , "PhysicalType": "DOUBLE" },
- { "ConvertedType": "NONE", "Id": "8" , "LogicalType": { "Type": "None"
}, "Name": "date_string_col", "PhysicalType": "BYTE_ARRAY" },
- { "ConvertedType": "NONE", "Id": "9" , "LogicalType": { "Type": "None"
}, "Name": "string_col" , "PhysicalType": "BYTE_ARRAY" },
- { "ConvertedType": "NONE", "Id": "10", "LogicalType": { "Type": "None"
}, "Name": "timestamp_col" , "PhysicalType": "INT96" }
+ { "Id": "0", "Name": "id", "PhysicalType": "INT32", "ConvertedType":
"NONE", "LogicalType": {"Type": "None"} },
+ {
+ "Id": "1",
+ "Name": "bool_col",
+ "PhysicalType": "BOOLEAN",
+ "ConvertedType": "NONE",
+ "LogicalType": {"Type": "None"}
+ },
+ {
+ "Id": "2",
+ "Name": "tinyint_col",
+ "PhysicalType": "INT32",
+ "ConvertedType": "NONE",
+ "LogicalType": {"Type": "None"}
+ },
+ {
+ "Id": "3",
+ "Name": "smallint_col",
+ "PhysicalType": "INT32",
+ "ConvertedType": "NONE",
+ "LogicalType": {"Type": "None"}
+ },
+ {
+ "Id": "4",
+ "Name": "int_col",
+ "PhysicalType": "INT32",
+ "ConvertedType": "NONE",
+ "LogicalType": {"Type": "None"}
+ },
+ {
+ "Id": "5",
+ "Name": "bigint_col",
+ "PhysicalType": "INT64",
+ "ConvertedType": "NONE",
+ "LogicalType": {"Type": "None"}
+ },
+ {
+ "Id": "6",
+ "Name": "float_col",
+ "PhysicalType": "FLOAT",
+ "ConvertedType": "NONE",
+ "LogicalType": {"Type": "None"}
+ },
+ {
+ "Id": "7",
+ "Name": "double_col",
+ "PhysicalType": "DOUBLE",
+ "ConvertedType": "NONE",
+ "LogicalType": {"Type": "None"}
+ },
+ {
+ "Id": "8",
+ "Name": "date_string_col",
+ "PhysicalType": "BYTE_ARRAY",
+ "ConvertedType": "NONE",
+ "LogicalType": {"Type": "None"}
+ },
+ {
+ "Id": "9",
+ "Name": "string_col",
+ "PhysicalType": "BYTE_ARRAY",
+ "ConvertedType": "NONE",
+ "LogicalType": {"Type": "None"}
+ },
+ {
+ "Id": "10",
+ "Name": "timestamp_col",
+ "PhysicalType": "INT96",
+ "ConvertedType": "NONE",
+ "LogicalType": {"Type": "None"}
+ }
],
"RowGroups": [
{
@@ -1219,17 +1283,105 @@ TEST_F(TestJSONWithLocalFile, JSONOutput) {
"TotalCompressedBytes": "0",
"Rows": "8",
"ColumnChunks": [
- { "CompressedSize": "73" , "Compression": "UNCOMPRESSED",
"Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "Id": "0" , "StatsSet": "False",
"UncompressedSize": "73" , "Values": "8" },
- { "CompressedSize": "24" , "Compression": "UNCOMPRESSED",
"Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "Id": "1" , "StatsSet": "False",
"UncompressedSize": "24" , "Values": "8" },
- { "CompressedSize": "47" , "Compression": "UNCOMPRESSED",
"Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "Id": "2" , "StatsSet": "False",
"UncompressedSize": "47" , "Values": "8" },
- { "CompressedSize": "47" , "Compression": "UNCOMPRESSED",
"Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "Id": "3" , "StatsSet": "False",
"UncompressedSize": "47" , "Values": "8" },
- { "CompressedSize": "47" , "Compression": "UNCOMPRESSED",
"Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "Id": "4" , "StatsSet": "False",
"UncompressedSize": "47" , "Values": "8" },
- { "CompressedSize": "55" , "Compression": "UNCOMPRESSED",
"Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "Id": "5" , "StatsSet": "False",
"UncompressedSize": "55" , "Values": "8" },
- { "CompressedSize": "47" , "Compression": "UNCOMPRESSED",
"Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "Id": "6" , "StatsSet": "False",
"UncompressedSize": "47" , "Values": "8" },
- { "CompressedSize": "55" , "Compression": "UNCOMPRESSED",
"Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "Id": "7" , "StatsSet": "False",
"UncompressedSize": "55" , "Values": "8" },
- { "CompressedSize": "88" , "Compression": "UNCOMPRESSED",
"Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "Id": "8" , "StatsSet": "False",
"UncompressedSize": "88" , "Values": "8" },
- { "CompressedSize": "49" , "Compression": "UNCOMPRESSED",
"Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "Id": "9" , "StatsSet": "False",
"UncompressedSize": "49" , "Values": "8" },
- { "CompressedSize": "139", "Compression": "UNCOMPRESSED",
"Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "Id": "10", "StatsSet": "False",
"UncompressedSize": "139", "Values": "8" }
+ {
+ "Id": "0",
+ "Values": "8",
+ "StatsSet": "False",
+ "Compression": "UNCOMPRESSED",
+ "Encodings": "RLE PLAIN_DICTIONARY PLAIN ",
+ "UncompressedSize": "73",
+ "CompressedSize": "73"
+ },
+ {
+ "Id": "1",
+ "Values": "8",
+ "StatsSet": "False",
+ "Compression": "UNCOMPRESSED",
+ "Encodings": "RLE PLAIN_DICTIONARY PLAIN ",
+ "UncompressedSize": "24",
+ "CompressedSize": "24"
+ },
+ {
+ "Id": "2",
+ "Values": "8",
+ "StatsSet": "False",
+ "Compression": "UNCOMPRESSED",
+ "Encodings": "RLE PLAIN_DICTIONARY PLAIN ",
+ "UncompressedSize": "47",
+ "CompressedSize": "47"
+ },
+ {
+ "Id": "3",
+ "Values": "8",
+ "StatsSet": "False",
+ "Compression": "UNCOMPRESSED",
+ "Encodings": "RLE PLAIN_DICTIONARY PLAIN ",
+ "UncompressedSize": "47",
+ "CompressedSize": "47"
+ },
+ {
+ "Id": "4",
+ "Values": "8",
+ "StatsSet": "False",
+ "Compression": "UNCOMPRESSED",
+ "Encodings": "RLE PLAIN_DICTIONARY PLAIN ",
+ "UncompressedSize": "47",
+ "CompressedSize": "47"
+ },
+ {
+ "Id": "5",
+ "Values": "8",
+ "StatsSet": "False",
+ "Compression": "UNCOMPRESSED",
+ "Encodings": "RLE PLAIN_DICTIONARY PLAIN ",
+ "UncompressedSize": "55",
+ "CompressedSize": "55"
+ },
+ {
+ "Id": "6",
+ "Values": "8",
+ "StatsSet": "False",
+ "Compression": "UNCOMPRESSED",
+ "Encodings": "RLE PLAIN_DICTIONARY PLAIN ",
+ "UncompressedSize": "47",
+ "CompressedSize": "47"
+ },
+ {
+ "Id": "7",
+ "Values": "8",
+ "StatsSet": "False",
+ "Compression": "UNCOMPRESSED",
+ "Encodings": "RLE PLAIN_DICTIONARY PLAIN ",
+ "UncompressedSize": "55",
+ "CompressedSize": "55"
+ },
+ {
+ "Id": "8",
+ "Values": "8",
+ "StatsSet": "False",
+ "Compression": "UNCOMPRESSED",
+ "Encodings": "RLE PLAIN_DICTIONARY PLAIN ",
+ "UncompressedSize": "88",
+ "CompressedSize": "88"
+ },
+ {
+ "Id": "9",
+ "Values": "8",
+ "StatsSet": "False",
+ "Compression": "UNCOMPRESSED",
+ "Encodings": "RLE PLAIN_DICTIONARY PLAIN ",
+ "UncompressedSize": "49",
+ "CompressedSize": "49"
+ },
+ {
+ "Id": "10",
+ "Values": "8",
+ "StatsSet": "False",
+ "Compression": "UNCOMPRESSED",
+ "Encodings": "RLE PLAIN_DICTIONARY PLAIN ",
+ "UncompressedSize": "139",
+ "CompressedSize": "139"
+ }
]
}
]
@@ -1243,6 +1395,11 @@ TEST_F(TestJSONWithLocalFile, JSONOutput) {
TEST_F(TestJSONWithLocalFile, JSONOutputFLBA) {
// min-max stats for FLBA contains non-utf8 output, so we don't check
// the whole json output.
+ // simdjson 5.0 changed the format of fractured_json
+ if constexpr (simdjson::SIMDJSON_VERSION_MAJOR < 5) {
+ GTEST_SKIP() << "Test requires simdjson >= 5";
+ }
+
std::string json_content =
ReadFromLocalFile("fixed_length_byte_array.parquet");
std::string json_contains = R"###({
@@ -1259,7 +1416,7 @@ TEST_F(TestJSONWithLocalFile, JSONOutputFLBA) {
"Name": "flba_field",
"PhysicalType": "FIXED_LEN_BYTE_ARRAY(4)",
"ConvertedType": "NONE",
- "LogicalType": { "Type": "None" }
+ "LogicalType": {"Type": "None"}
}
],)###";
@@ -1267,11 +1424,18 @@ TEST_F(TestJSONWithLocalFile, JSONOutputFLBA) {
}
TEST_F(TestJSONWithLocalFile, JSONOutputSortColumns) {
+ // simdjson 5.0 changed the format of fractured_json
+ if constexpr (simdjson::SIMDJSON_VERSION_MAJOR < 5) {
+ GTEST_SKIP() << "Test requires simdjson >= 5";
+ }
+
std::string json_content = ReadFromLocalFile("sort_columns.parquet");
std::string json_contains = R"###("SortColumns": [
- { "column_idx": 0, "descending": 1, "nulls_first": 1 }, {
"column_idx": 1, "descending": 0, "nulls_first": 0 }
+ {"column_idx": 0, "descending": 1, "nulls_first": 1},
+ {"column_idx": 1, "descending": 0, "nulls_first": 0}
],)###";
+
EXPECT_THAT(json_content, testing::HasSubstr(json_contains));
}
