wgtmac commented on code in PR #51357:
URL: https://github.com/apache/arrow/pull/51357#discussion_r4034109928
##########
cpp/src/parquet/column_writer.cc:
##########
@@ -1750,7 +1750,8 @@ class TypedColumnWriterImpl : public ColumnWriterImpl,
internal::DefLevelsToBitmap(def_levels, batch_size, level_info_, &io);
*out_values_to_write = io.values_read - io.null_count;
*out_spaced_values_to_write = io.values_read;
- *null_count = io.null_count;
+ // Include nulls from repeated ancestors, which are excluded from
io.null_count.
Review Comment:
This comment will be misleading for readers do not know this PR.
##########
cpp/src/parquet/arrow/arrow_statistics_test.cc:
##########
@@ -160,6 +160,50 @@ INSTANTIATE_TEST_SUITE_P(
/*expected_min=*/"z",
/*expected_max=*/"z"}));
+TEST(StatisticsTest, FixedWidthLeafUnderListStructNullCount) {
+ // Null counts for fixed-width leaves under list<struct<...>>
+ // must include null and empty list entries from the repeated ancestor.
+ auto schema = ::arrow::schema({::arrow::field(
+ "col", ::arrow::list(::arrow::struct_(
+ {::arrow::field("s", ::arrow::utf8()),
+ ::arrow::field("i32", ::arrow::int32())})))});
+
+ auto table = Table::Make(
+ schema,
+ {ArrayFromJSON(
Review Comment:
Can we use `TableFromJson` directly?
##########
cpp/src/parquet/arrow/arrow_statistics_test.cc:
##########
@@ -160,6 +160,50 @@ INSTANTIATE_TEST_SUITE_P(
/*expected_min=*/"z",
/*expected_max=*/"z"}));
+TEST(StatisticsTest, FixedWidthLeafUnderListStructNullCount) {
+ // Null counts for fixed-width leaves under list<struct<...>>
+ // must include null and empty list entries from the repeated ancestor.
+ auto schema = ::arrow::schema({::arrow::field(
+ "col", ::arrow::list(::arrow::struct_(
+ {::arrow::field("s", ::arrow::utf8()),
+ ::arrow::field("i32", ::arrow::int32())})))});
+
+ auto table = Table::Make(
+ schema,
+ {ArrayFromJSON(
+ schema->field(0)->type(),
+
R"([[{"s":"a","i32":1}],null,[],[{"s":null,"i32":null},{"s":"b","i32":2}]])")});
+
+ std::shared_ptr<::arrow::ResizableBuffer> serialized_data = AllocateBuffer();
+ auto out_stream =
+ std::make_shared<::arrow::io::BufferOutputStream>(serialized_data);
+
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FileWriter> writer,
+ FileWriter::Open(*schema, default_memory_pool(), out_stream,
+ default_writer_properties(),
+ default_arrow_writer_properties()));
+ ASSERT_OK(writer->WriteTable(*table, std::numeric_limits<int64_t>::max()));
Review Comment:
```suggestion
ASSERT_OK(writer->WriteTable(*table));
```
##########
cpp/src/parquet/arrow/arrow_statistics_test.cc:
##########
@@ -160,6 +160,50 @@ INSTANTIATE_TEST_SUITE_P(
/*expected_min=*/"z",
/*expected_max=*/"z"}));
+TEST(StatisticsTest, FixedWidthLeafUnderListStructNullCount) {
+ // Null counts for fixed-width leaves under list<struct<...>>
+ // must include null and empty list entries from the repeated ancestor.
+ auto schema = ::arrow::schema({::arrow::field(
+ "col", ::arrow::list(::arrow::struct_(
+ {::arrow::field("s", ::arrow::utf8()),
+ ::arrow::field("i32", ::arrow::int32())})))});
+
+ auto table = Table::Make(
+ schema,
+ {ArrayFromJSON(
+ schema->field(0)->type(),
+
R"([[{"s":"a","i32":1}],null,[],[{"s":null,"i32":null},{"s":"b","i32":2}]])")});
+
+ std::shared_ptr<::arrow::ResizableBuffer> serialized_data = AllocateBuffer();
+ auto out_stream =
+ std::make_shared<::arrow::io::BufferOutputStream>(serialized_data);
+
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FileWriter> writer,
+ FileWriter::Open(*schema, default_memory_pool(), out_stream,
+ default_writer_properties(),
+ default_arrow_writer_properties()));
+ ASSERT_OK(writer->WriteTable(*table, std::numeric_limits<int64_t>::max()));
+ ASSERT_OK(writer->Close());
+ ASSERT_OK(out_stream->Close());
+
+ auto buffer_reader =
std::make_shared<::arrow::io::BufferReader>(serialized_data);
+ auto parquet_reader = ParquetFileReader::Open(std::move(buffer_reader));
+ auto metadata = parquet_reader->metadata();
+ auto row_group = metadata->RowGroup(0);
+
+ ASSERT_EQ(row_group->num_columns(), 2);
+
+ auto int32_stats = row_group->ColumnChunk(1)->statistics();
+
+ ASSERT_NE(int32_stats, nullptr);
+
+ // Fixed-width leaves must include nulls from repeated ancestors
+ // (e.g. null or empty lists) in the column statistics.
Review Comment:
```suggestion
// Fixed-width leaves must include nulls from repeated ancestors
// (e.g. null or empty lists) in the column statistics.
auto int32_stats = row_group->ColumnChunk(1)->statistics();
ASSERT_NE(int32_stats, nullptr);
```
A few stylish fixes.
##########
cpp/src/parquet/arrow/arrow_statistics_test.cc:
##########
@@ -160,6 +160,50 @@ INSTANTIATE_TEST_SUITE_P(
/*expected_min=*/"z",
/*expected_max=*/"z"}));
+TEST(StatisticsTest, FixedWidthLeafUnderListStructNullCount) {
+ // Null counts for fixed-width leaves under list<struct<...>>
+ // must include null and empty list entries from the repeated ancestor.
+ auto schema = ::arrow::schema({::arrow::field(
+ "col", ::arrow::list(::arrow::struct_(
+ {::arrow::field("s", ::arrow::utf8()),
+ ::arrow::field("i32", ::arrow::int32())})))});
+
+ auto table = Table::Make(
+ schema,
+ {ArrayFromJSON(
+ schema->field(0)->type(),
+
R"([[{"s":"a","i32":1}],null,[],[{"s":null,"i32":null},{"s":"b","i32":2}]])")});
+
+ std::shared_ptr<::arrow::ResizableBuffer> serialized_data = AllocateBuffer();
+ auto out_stream =
+ std::make_shared<::arrow::io::BufferOutputStream>(serialized_data);
+
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FileWriter> writer,
+ FileWriter::Open(*schema, default_memory_pool(), out_stream,
+ default_writer_properties(),
+ default_arrow_writer_properties()));
+ ASSERT_OK(writer->WriteTable(*table, std::numeric_limits<int64_t>::max()));
+ ASSERT_OK(writer->Close());
+ ASSERT_OK(out_stream->Close());
+
+ auto buffer_reader =
std::make_shared<::arrow::io::BufferReader>(serialized_data);
+ auto parquet_reader = ParquetFileReader::Open(std::move(buffer_reader));
+ auto metadata = parquet_reader->metadata();
+ auto row_group = metadata->RowGroup(0);
+
+ ASSERT_EQ(row_group->num_columns(), 2);
Review Comment:
```suggestion
auto row_group = metadata->RowGroup(0);
ASSERT_EQ(row_group->num_columns(), 2);
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
