justinli500 commented on code in PR #49855:
URL: https://github.com/apache/arrow/pull/49855#discussion_r4520953366
##########
cpp/src/parquet/arrow/arrow_reader_writer_test.cc:
##########
@@ -2726,6 +2728,201 @@ TEST(TestArrowReadWrite, GetRecordBatchReaderNoColumns)
{
ASSERT_EQ(actual_batch->num_rows(), num_rows);
}
+// GH-39808: bytes cached by PreBuffer() for a decoded row group must be
+// releasable, else Dataset.to_batches accumulates memory over the reader's
life.
+TEST(TestArrowReadWrite, EvictPreBufferedDataBefore) {
+ ArrowReaderProperties properties = default_arrow_reader_properties();
+ properties.set_pre_buffer(true);
+ const int num_rows = 1024;
+ const int row_group_size = 256;
+ const int num_columns = 3;
+ const std::vector<int> row_groups = {0, 1, 2, 3};
+ const std::vector<int> column_indices = {0, 1, 2};
+
+ std::shared_ptr<Table> table;
+ ASSERT_NO_FATAL_FAILURE(MakeDoubleTable(num_columns, num_rows, 1, &table));
+
+ std::shared_ptr<Buffer> buffer;
+ ASSERT_NO_FATAL_FAILURE(WriteTableToBuffer(table, row_group_size,
+
default_arrow_writer_properties(), &buffer));
+
+ std::unique_ptr<FileReader> reader;
+ FileReaderBuilder builder;
+ ASSERT_OK(builder.Open(std::make_shared<BufferReader>(buffer)));
+ ASSERT_OK(builder.properties(properties)->Build(&reader));
+ ASSERT_EQ(reader->num_row_groups(), static_cast<int>(row_groups.size()));
+
+ reader->parquet_reader()->PreBuffer(row_groups, column_indices,
+ ::arrow::io::IOContext(),
+
::arrow::io::CacheOptions::LazyDefaults());
+ ASSERT_OK(reader->parquet_reader()->WhenBuffered(row_groups,
column_indices).status());
+
+ std::shared_ptr<Table> rg_table;
+ ASSERT_OK(reader->ReadRowGroup(/*i=*/0, column_indices, &rg_table));
Review Comment:
Good catch - I disabled automatic prebuffering in this test so each read
preserves the cache created by the explicit `PreBuffer()` call. Now it checks
that upcoming groups remain cached after eviction.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
