This is an automated email from the ASF dual-hosted git repository. mgrigorov pushed a commit to branch branch-1.11 in repository https://gitbox.apache.org/repos/asf/avro.git
commit f66f7470a982f706e812dedbc66abba530ff82e5 Author: Martin Grigorov <[email protected]> AuthorDate: Thu Jan 13 10:22:15 2022 +0200 AVRO-3303: Rust: Add support for Xz codec (#1455) Signed-off-by: Martin Tzvetanov Grigorov <[email protected]> (cherry picked from commit ee47d4cd64eddcbced7b7c77102480e767b210a6) --- lang/rust/Cargo.toml | 2 ++ lang/rust/README.md | 12 ++++++++++ lang/rust/src/codec.rs | 62 ++++++++++++++++++++++++++++++++------------------ 3 files changed, 54 insertions(+), 22 deletions(-) diff --git a/lang/rust/Cargo.toml b/lang/rust/Cargo.toml index 5fb8593..0f1f4ed 100644 --- a/lang/rust/Cargo.toml +++ b/lang/rust/Cargo.toml @@ -32,6 +32,7 @@ documentation = "https://docs.rs/avro-rs"; snappy = ["crc32fast", "snap"] zstandard = ["zstd"] bzip = ["bzip2"] +xz = ["xz2"] [lib] path = "src/lib.rs" @@ -56,6 +57,7 @@ bzip2 = { version = "0.4.3", optional = true } crc32fast = { version = "1.2.1", optional = true } digest = "0.10.1" libflate = "1.1.1" +xz2 = { version = "0.1.6", optional = true } num-bigint = "0.4.2" rand = "0.8.4" regex = "1.5.4" diff --git a/lang/rust/README.md b/lang/rust/README.md index 93950ca..0282d51 100644 --- a/lang/rust/README.md +++ b/lang/rust/README.md @@ -87,6 +87,15 @@ version = "x.y" features = ["bzip"] ``` +Or in case you want to leverage the **Xz** codec: + +```toml +[dependencies.avro-rs] +version = "x.y" +features = ["xz"] +``` + + ## Upgrading to a newer minor version The library is still in beta, so there might be backward-incompatible changes between minor @@ -264,6 +273,9 @@ the block. You must enable the `snappy` feature to use this codec. You must enable the `zstandard` feature to use this codec. * **Bzip2**: uses [BZip2](https://sourceware.org/bzip2/) compression library. You must enable the `bzip` feature to use this codec. +* **Xz**: uses [xz2](https://github.com/alexcrichton/xz2-rs) compression library. + You must enable the `xz` feature to use this codec. + To specify a codec to use to compress data, just specify it while creating a `Writer`: ```rust diff --git a/lang/rust/src/codec.rs b/lang/rust/src/codec.rs index 15992c1..0ba8abe 100644 --- a/lang/rust/src/codec.rs +++ b/lang/rust/src/codec.rs @@ -30,6 +30,8 @@ use bzip2::{ extern crate crc32fast; #[cfg(feature = "snappy")] use crc32fast::Hasher; +#[cfg(feature = "xz")] +use xz2::read::{XzDecoder, XzEncoder}; /// The compression codec used to compress blocks. #[derive(Clone, Copy, Debug, PartialEq, EnumString, IntoStaticStr)] @@ -52,6 +54,10 @@ pub enum Codec { /// The `BZip2` codec uses [BZip2](https://sourceware.org/bzip2/) /// compression library. Bzip2, + #[cfg(feature = "xz")] + /// The `Xz` codec uses [Xz utils](https://tukaani.org/xz/) + /// compression library. + Xz, } impl From<Codec> for Value { @@ -104,6 +110,14 @@ impl Codec { encoder.read_to_end(&mut buffer).unwrap(); *stream = buffer; } + #[cfg(feature = "xz")] + Codec::Xz => { + let compression_level = 9; + let mut encoder = XzEncoder::new(&stream[..], compression_level); + let mut buffer = Vec::new(); + encoder.read_to_end(&mut buffer).unwrap(); + *stream = buffer; + } }; Ok(()) @@ -156,6 +170,13 @@ impl Codec { decoder.read_to_end(&mut decoded).unwrap(); decoded } + #[cfg(feature = "xz")] + Codec::Xz => { + let mut decoder = XzDecoder::new(&stream[..]); + let mut decoded: Vec<u8> = Vec::new(); + decoder.read_to_end(&mut decoded).unwrap(); + decoded + } }; Ok(()) } @@ -179,43 +200,34 @@ mod tests { #[test] fn deflate_compress_and_decompress() { - let codec = Codec::Deflate; - let mut stream = INPUT.to_vec(); - codec.compress(&mut stream).unwrap(); - assert_ne!(INPUT, stream.as_slice()); - assert!(INPUT.len() > stream.len()); - codec.decompress(&mut stream).unwrap(); - assert_eq!(INPUT, stream.as_slice()); + compress_and_decompress(Codec::Deflate); } #[cfg(feature = "snappy")] #[test] fn snappy_compress_and_decompress() { - let codec = Codec::Snappy; - let mut stream = INPUT.to_vec(); - codec.compress(&mut stream).unwrap(); - assert_ne!(INPUT, stream.as_slice()); - assert!(INPUT.len() > stream.len()); - codec.decompress(&mut stream).unwrap(); - assert_eq!(INPUT, stream.as_slice()); + compress_and_decompress(Codec::Snappy); } #[cfg(feature = "zstandard")] #[test] fn zstd_compress_and_decompress() { - let codec = Codec::Zstd; - let mut stream = INPUT.to_vec(); - codec.compress(&mut stream).unwrap(); - assert_ne!(INPUT, stream.as_slice()); - assert!(INPUT.len() > stream.len()); - codec.decompress(&mut stream).unwrap(); - assert_eq!(INPUT, stream.as_slice()); + compress_and_decompress(Codec::Zstd); } #[cfg(feature = "bzip")] #[test] fn bzip_compress_and_decompress() { - let codec = Codec::Bzip2; + compress_and_decompress(Codec::Bzip2); + } + + #[cfg(feature = "xz")] + #[test] + fn xz_compress_and_decompress() { + compress_and_decompress(Codec::Xz); + } + + fn compress_and_decompress(codec: Codec) { let mut stream = INPUT.to_vec(); codec.compress(&mut stream).unwrap(); assert_ne!(INPUT, stream.as_slice()); @@ -237,6 +249,9 @@ mod tests { #[cfg(feature = "bzip")] assert_eq!(<&str>::from(Codec::Bzip2), "bzip2"); + + #[cfg(feature = "xz")] + assert_eq!(<&str>::from(Codec::Xz), "xz"); } #[test] @@ -255,6 +270,9 @@ mod tests { #[cfg(feature = "bzip")] assert_eq!(Codec::from_str("bzip2").unwrap(), Codec::Bzip2); + #[cfg(feature = "xz")] + assert_eq!(Codec::from_str("xz").unwrap(), Codec::Xz); + assert!(Codec::from_str("not a codec").is_err()); } }
