gabotechs commented on code in PR #12084:
URL: https://github.com/apache/arrow-rs/pull/12084#discussion_r4490140486
##########
arrow-ipc/src/compression.rs:
##########
@@ -275,19 +290,68 @@ impl CompressionCodec {
}
}
+/// Reusable lz4 frame compressor, the lz4 counterpart of
`zstd::bulk::Compressor`.
+///
+/// `lz4_flex` has no bulk frame API, so this wraps a
[`lz4_flex::frame::FrameEncoder`] whose hash
+/// table and block buffers are retained across calls to
[`Self::compress_to_buffer`].
#[cfg(feature = "lz4")]
-fn compress_lz4(input: &[u8], output: &mut Vec<u8>) -> Result<(), ArrowError> {
- use std::io::Write;
- let mut encoder = lz4_flex::frame::FrameEncoder::new(output);
- encoder.write_all(input)?;
- encoder
- .finish()
- .map_err(|e| ArrowError::ExternalError(Box::new(e)))?;
- Ok(())
+struct Lz4Compressor {
+ encoder: lz4_flex::frame::FrameEncoder<Vec<u8>>,
+}
+
+#[cfg(feature = "lz4")]
+impl Lz4Compressor {
+ fn new() -> Self {
+ // Pin the block size: with `BlockSize::Auto` the first buffer written
would fix it for
+ // every subsequent frame of the reused encoder. 256KB keeps large
buffers as fast as
+ // with bigger blocks while the encoder retains only ~0.5MB of buffers.
+ let frame_info =
+
lz4_flex::frame::FrameInfo::new().block_size(lz4_flex::frame::BlockSize::Max256KB);
Review Comment:
I didn't make it configurable because this is what proved best in the
benchmarks and because it'd require a breaking change (major release) instead
of being suitable for just a minor release.
https://github.com/apache/arrow-rs/blob/main/arrow-ipc/src/compression.rs#L128-L130
If you think it's worth it I can do it though.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
