Rich-T-kid commented on code in PR #12084:
URL: https://github.com/apache/arrow-rs/pull/12084#discussion_r4493462754
##########
arrow-ipc/src/compression.rs:
##########
@@ -275,19 +290,68 @@ impl CompressionCodec {
}
}
+/// Reusable lz4 frame compressor, the lz4 counterpart of
`zstd::bulk::Compressor`.
+///
+/// `lz4_flex` has no bulk frame API, so this wraps a
[`lz4_flex::frame::FrameEncoder`] whose hash
+/// table and block buffers are retained across calls to
[`Self::compress_to_buffer`].
#[cfg(feature = "lz4")]
-fn compress_lz4(input: &[u8], output: &mut Vec<u8>) -> Result<(), ArrowError> {
- use std::io::Write;
- let mut encoder = lz4_flex::frame::FrameEncoder::new(output);
- encoder.write_all(input)?;
- encoder
- .finish()
- .map_err(|e| ArrowError::ExternalError(Box::new(e)))?;
- Ok(())
+struct Lz4Compressor {
+ encoder: lz4_flex::frame::FrameEncoder<Vec<u8>>,
+}
+
+#[cfg(feature = "lz4")]
+impl Lz4Compressor {
+ fn new() -> Self {
+ // Pin the block size: with `BlockSize::Auto` the first buffer written
would fix it for
+ // every subsequent frame of the reused encoder. 256KB keeps large
buffers as fast as
+ // with bigger blocks while the encoder retains only ~0.5MB of buffers.
+ let frame_info =
+
lz4_flex::frame::FrameInfo::new().block_size(lz4_flex::frame::BlockSize::Max256KB);
Review Comment:
no I don't think its strictly necessary. Ill create a follow up issue and
tag it with the next major release
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
