Rich-T-kid commented on code in PR #12077:
URL: https://github.com/apache/arrow-rs/pull/12077#discussion_r4475648037
##########
arrow-ipc/src/compression.rs:
##########
@@ -308,21 +308,7 @@ fn compress_lz4(_input: &[u8], _output: &mut Vec<u8>) ->
Result<(), ArrowError>
#[cfg(feature = "lz4")]
fn decompress_lz4(input: &[u8], decompressed_size: usize) -> Result<Vec<u8>,
ArrowError> {
- use std::io::Read;
- let mut output = Vec::with_capacity(decompressed_size);
- let mut decoder = lz4_flex::frame::FrameDecoder::new(input);
- decoder
- .by_ref()
- .take(decompressed_size as u64)
- .read_to_end(&mut output)?;
-
- // Probe without growing `output` to reject data exceeding the advertised
size.
- if decoder.read(&mut [0])? != 0 {
- return Err(ArrowError::IpcError(format!(
- "LZ4 decompressed buffer exceeds advertised size of
{decompressed_size}"
- )));
- }
- Ok(output)
+ read_to_end_bounded(lz4_flex::frame::FrameDecoder::new(input),
decompressed_size)
Review Comment:
This can stay much closer to the existing implementation. Only the initial
allocation needs to stop trusting the sender-provided size; `read_to_end` will
then grow with actual decompressed output, while `take` and the probe preserve
the existing size checks.
```suggestion
use std::io::Read;
let mut output = Vec::with_capacity(
decompressed_size.min(crate::reader::MAX_PREALLOC_BYTES),
);
let mut decoder = lz4_flex::frame::FrameDecoder::new(input);
decoder
.by_ref()
.take(decompressed_size as u64)
.read_to_end(&mut output)?;
// Probe without growing `output` to reject data exceeding the
advertised size.
if decoder.read(&mut [0])? != 0 {
return Err(ArrowError::IpcError(format!(
"LZ4 decompressed buffer exceeds advertised size of
{decompressed_size}"
)));
}
Ok(output)
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
