This is an automated email from the ASF dual-hosted git repository.
Jefffrey pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/main by this push:
new 9d0e6c760c feat(parquet): Support negative ZSTD levels (#10601)
9d0e6c760c is described below
commit 9d0e6c760c9dcbe3fc93d486981dca9c81f2337e
Author: Nuno Faria <[email protected]>
AuthorDate: Mon Aug 10 13:25:51 2026 +0100
feat(parquet): Support negative ZSTD levels (#10601)
# Which issue does this PR close?
- Closes #10560.
# Rationale for this change
Allow Parquet files to be compressed with negative ZSTD levels, which
are allowed in `libzstd`.
# What changes are included in this PR?
The minimum ZSTD level allowed is now `-131072` (see the issue for more
details).
The `split_compression_string` function has also been updated to allow
negative numbers.
# Are these changes tested?
Yes.
# Are there any user-facing changes?
Negative ZSTD levels are now accepted.
---
parquet/src/basic.rs | 16 +++++++++-------
parquet/src/compression.rs | 8 ++++++--
2 files changed, 15 insertions(+), 9 deletions(-)
diff --git a/parquet/src/basic.rs b/parquet/src/basic.rs
index 4458af160d..ab99f7b682 100644
--- a/parquet/src/basic.rs
+++ b/parquet/src/basic.rs
@@ -708,13 +708,13 @@ impl From<Compression> for CompressionCodec {
}
}
-fn split_compression_string(str_setting: &str) -> Result<(&str, Option<u32>),
ParquetError> {
+fn split_compression_string(str_setting: &str) -> Result<(&str, Option<i32>),
ParquetError> {
let split_setting = str_setting.split_once('(');
match split_setting {
Some((codec, level_str)) => {
let level = &level_str[..level_str.len() - 1]
- .parse::<u32>()
+ .parse::<i32>()
.map_err(|_| {
ParquetError::General(format!("invalid compression level:
{level_str}"))
})?;
@@ -724,7 +724,7 @@ fn split_compression_string(str_setting: &str) ->
Result<(&str, Option<u32>), Pa
}
}
-fn check_level_is_none(level: &Option<u32>) -> Result<(), ParquetError> {
+fn check_level_is_none(level: &Option<i32>) -> Result<(), ParquetError> {
if level.is_some() {
return Err(ParquetError::General(
"compression level is not supported".to_string(),
@@ -734,7 +734,7 @@ fn check_level_is_none(level: &Option<u32>) -> Result<(),
ParquetError> {
Ok(())
}
-fn require_level(codec: &str, level: Option<u32>) -> Result<u32, ParquetError>
{
+fn require_level(codec: &str, level: Option<i32>) -> Result<i32, ParquetError>
{
level.ok_or(ParquetError::General(format!(
"{codec} requires a compression level",
)))
@@ -757,7 +757,7 @@ impl FromStr for Compression {
}
"GZIP" | "gzip" => {
let level = require_level(codec, level)?;
- Compression::GZIP(GzipLevel::try_new(level)?)
+ Compression::GZIP(GzipLevel::try_new(level.try_into()?)?)
}
"LZO" | "lzo" => {
check_level_is_none(&level)?;
@@ -765,7 +765,7 @@ impl FromStr for Compression {
}
"BROTLI" | "brotli" => {
let level = require_level(codec, level)?;
- Compression::BROTLI(BrotliLevel::try_new(level)?)
+ Compression::BROTLI(BrotliLevel::try_new(level.try_into()?)?)
}
"LZ4" | "lz4" => {
check_level_is_none(&level)?;
@@ -773,7 +773,7 @@ impl FromStr for Compression {
}
"ZSTD" | "zstd" => {
let level = require_level(codec, level)?;
- Compression::ZSTD(ZstdLevel::try_new(level as i32)?)
+ Compression::ZSTD(ZstdLevel::try_new(level)?)
}
"LZ4_RAW" | "lz4_raw" => {
check_level_is_none(&level)?;
@@ -2334,6 +2334,8 @@ mod tests {
assert_eq!(compress, Compression::LZO);
compress = "zstd(3)".parse().unwrap();
assert_eq!(compress,
Compression::ZSTD(ZstdLevel::try_new(3).unwrap()));
+ compress = "zstd(-3)".parse().unwrap();
+ assert_eq!(compress,
Compression::ZSTD(ZstdLevel::try_new(-3).unwrap()));
compress = "LZ4_RAW".parse().unwrap();
assert_eq!(compress, Compression::LZ4_RAW);
compress = "uncompressed".parse().unwrap();
diff --git a/parquet/src/compression.rs b/parquet/src/compression.rs
index cb1620e3fe..55783de1c3 100644
--- a/parquet/src/compression.rs
+++ b/parquet/src/compression.rs
@@ -579,7 +579,7 @@ pub struct ZstdLevel(i32);
impl CompressionLevel<i32> for ZstdLevel {
// zstd binds to C, and hence zstd::compression_level_range() is not const
as this calls the
// underlying C library.
- const MINIMUM_LEVEL: i32 = 1;
+ const MINIMUM_LEVEL: i32 = -131072;
const MAXIMUM_LEVEL: i32 = 22;
}
@@ -935,7 +935,11 @@ mod tests {
#[test]
fn test_codec_zstd() {
- for level in ZstdLevel::MINIMUM_LEVEL..=ZstdLevel::MAXIMUM_LEVEL {
+ // since ZstdLevel::MINIMUM_LEVEL is a large negative number, we test
a smaller range
+ for level in [ZstdLevel::MINIMUM_LEVEL]
+ .into_iter()
+ .chain(-100..=ZstdLevel::MAXIMUM_LEVEL)
+ {
let level = ZstdLevel::try_new(level).unwrap();
test_codec_with_size(CodecType::ZSTD(level));
test_codec_without_size(CodecType::ZSTD(level));
