alamb commented on issue #7490:
URL: https://github.com/apache/arrow-rs/issues/7490#issuecomment-2923469096
Here is a fun reproducer
```rust
use std::io::BufWriter;
use std::sync::Arc;
use arrow::array::{ArrayRef, Int64Array, RecordBatch, StringViewArray};
use parquet::arrow::arrow_reader::ParquetRecordBatchReader;
use parquet::arrow::arrow_writer::ArrowWriterOptions;
use parquet::arrow::ArrowWriter;
use parquet::file::properties::WriterProperties;
fn main() {
let output= std::fs::File::create("output.parquet").unwrap();
let mut output = BufWriter::new(output);
let batch = make_batch('a');
let props = WriterProperties::builder()
.set_max_row_group_size(1)
.build();
let mut writer = ArrowWriter::try_new(&mut output, batch.schema(),
Some(props)).unwrap();
writer.write(&batch).unwrap();
for char in ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j'] {
let batch = make_batch(char);
writer.write(&batch).unwrap();
}
writer.close().unwrap();
drop(output);
println!("Hello, world!");
}
// Makes a batch with long string values for testing purposes.
fn make_batch(val: char) -> RecordBatch {
let col = Arc::new(StringViewArray::from_iter_values(
[val.to_string().repeat(100000)]
)) as ArrayRef;
RecordBatch::try_from_iter([("col", col)]).unwrap()
}
```
This results in a 3 MB parquet file of which 2.5MB is the statistics 😱
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
