Copilot commented on code in PR #55306:
URL: https://github.com/apache/arrow/pull/55306#discussion_r4408568868
##########
cpp/src/parquet/column_writer.cc:
##########
@@ -1004,22 +1004,26 @@ void ColumnWriterImpl::BuildDataPageV1(int64_t
definition_levels_rle_size,
int64_t repetition_levels_rle_size,
int64_t uncompressed_size,
const std::shared_ptr<Buffer>& values) {
- // Use Arrow::Buffer::shrink_to_fit = false
- // underlying buffer only keeps growing. Resize to a smaller size does not
reallocate.
- PARQUET_THROW_NOT_OK(uncompressed_data_->Resize(uncompressed_size, false));
- ConcatenateBuffers(definition_levels_rle_size, repetition_levels_rle_size,
values,
- uncompressed_data_->mutable_data());
+ const bool has_levels =
+ definition_levels_rle_size > 0 || repetition_levels_rle_size > 0;
+ if (has_levels) {
+ // Use Arrow::Buffer::shrink_to_fit = false
+ // underlying buffer only keeps growing. Resize to a smaller size does not
reallocate.
+ PARQUET_THROW_NOT_OK(uncompressed_data_->Resize(uncompressed_size, false));
+ ConcatenateBuffers(definition_levels_rle_size, repetition_levels_rle_size,
values,
+ uncompressed_data_->mutable_data());
+ }
auto [page_stats, page_size_stats] = GetPageStatistics();
page_stats.ApplyStatSizeLimits(properties_->max_statistics_size(descr_->path()));
page_stats.set_is_signed(SortOrder::SIGNED == descr_->sort_order());
ResetPageStatistics();
std::shared_ptr<Buffer> compressed_data;
if (pager_->has_compressor()) {
- pager_->Compress(*(uncompressed_data_.get()),
compressor_temp_buffer_.get());
+ pager_->Compress(*uncompressed_data_, compressor_temp_buffer_.get());
Review Comment:
For required, non-repeated columns, `has_levels` is false, so
`uncompressed_data_` is no longer populated with the encoded values. This call
still compresses that buffer instead of `values`, producing compressed V1 pages
whose contents do not match their declared uncompressed size. Select `values`
as the compression input when there are no levels, just as the uncompressed
branch does.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
