jorisvandenbossche commented on a change in pull request #7233:
URL: https://github.com/apache/arrow/pull/7233#discussion_r428283327
##########
File path: cpp/src/arrow/ipc/reader.cc
##########
@@ -347,45 +347,55 @@ Status DecompressBuffers(Compression::type compression,
const IpcReadOptions& op
std::unique_ptr<util::Codec> codec;
ARROW_ASSIGN_OR_RAISE(codec, util::Codec::Create(compression));
- auto DecompressOne = [&](int i) {
- ArrayData* arr = (*fields)[i].get();
- for (size_t i = 0; i < arr->buffers.size(); ++i) {
- if (arr->buffers[i] == nullptr) {
- continue;
- }
- if (arr->buffers[i]->size() == 0) {
- continue;
- }
- if (arr->buffers[i]->size() < 8) {
- return Status::Invalid(
- "Likely corrupted message, compressed buffers "
- "are larger than 8 bytes by construction");
- }
- const uint8_t* data = arr->buffers[i]->data();
- int64_t compressed_size = arr->buffers[i]->size() - sizeof(int64_t);
- int64_t uncompressed_size =
- BitUtil::FromLittleEndian(util::SafeLoadAs<int64_t>(data));
+ auto DecompressOneBuffer = [&](std::shared_ptr<Buffer>* buf) {
+ if ((*buf) == nullptr) {
+ return Status::OK();
+ }
+ if ((*buf)->size() == 0) {
+ return Status::OK();
+ }
+ if ((*buf)->size() < 8) {
+ return Status::Invalid(
+ "Likely corrupted message, compressed buffers "
+ "are larger than 8 bytes by construction");
+ }
+ const uint8_t* data = (*buf)->data();
+ int64_t compressed_size = (*buf)->size() - sizeof(int64_t);
+ int64_t uncompressed_size =
+ BitUtil::FromLittleEndian(util::SafeLoadAs<int64_t>(data));
- ARROW_ASSIGN_OR_RAISE(auto uncompressed,
- AllocateBuffer(uncompressed_size,
options.memory_pool));
+ ARROW_ASSIGN_OR_RAISE(auto uncompressed,
+ AllocateBuffer(uncompressed_size,
options.memory_pool));
- int64_t actual_decompressed;
- ARROW_ASSIGN_OR_RAISE(
- actual_decompressed,
- codec->Decompress(compressed_size, data + sizeof(int64_t),
uncompressed_size,
- uncompressed->mutable_data()));
- if (actual_decompressed != uncompressed_size) {
- return Status::Invalid("Failed to fully decompress buffer, expected ",
- uncompressed_size, " bytes but decompressed ",
- actual_decompressed);
+ int64_t actual_decompressed;
+ ARROW_ASSIGN_OR_RAISE(
+ actual_decompressed,
+ codec->Decompress(compressed_size, data + sizeof(int64_t),
uncompressed_size,
+ uncompressed->mutable_data()));
+ if (actual_decompressed != uncompressed_size) {
+ return Status::Invalid("Failed to fully decompress buffer, expected ",
+ uncompressed_size, " bytes but decompressed ",
+ actual_decompressed);
+ }
+ *buf = std::move(uncompressed);
+ return Status::OK();
+ };
+
+ auto DecompressOneField = [&](int i) {
+ ArrayData* arr = (*fields)[i].get();
+ for (size_t i = 0; i < arr->buffers.size(); ++i) {
+ ARROW_RETURN_NOT_OK(DecompressOneBuffer(&arr->buffers[i]));
+ }
+ for (size_t i = 0; i < arr->child_data.size(); ++i) {
+ for (size_t j = 0; j < arr->child_data[i]->buffers.size(); ++j) {
+
ARROW_RETURN_NOT_OK(DecompressOneBuffer(&arr->child_data[i]->buffers[j]));
Review comment:
TODO: on second thought, for multiple levels of nesting, this will need
to be a recursive loop for the child_data's child_data
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
