This is an automated email from the ASF dual-hosted git repository.
etseidl pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/main by this push:
new d40cac73f4 Fix: Error rather than panic on invalid dictionary index
bit width in Parquet reader (#10725)
d40cac73f4 is described below
commit d40cac73f4f6c65841d0619857b4bc81eeab8a5e
Author: Dhruv Vaishnav <[email protected]>
AuthorDate: Thu Aug 20 02:46:46 2026 +0530
Fix: Error rather than panic on invalid dictionary index bit width in
Parquet reader (#10725)
# Which issue does this PR close?
- Closes #10722.
# Rationale for this change
When reading dictionary-encoded Parquet data pages with DictIndexDecoder
or ByteArrayDictionaryReader, the first byte was read via data[0]
without checking if data is empty, and the index it_width was passed to
RleDecoder without checking the
---------
Co-authored-by: Ed Seidl <[email protected]>
---
.../arrow/array_reader/byte_array_dictionary.rs | 38 ++++++++++++++++++--
parquet/src/arrow/decoder/dictionary_index.rs | 41 ++++++++++++++++++++--
parquet/src/encodings/decoding.rs | 7 ++--
parquet/src/encodings/rle.rs | 11 ++++++
4 files changed, 88 insertions(+), 9 deletions(-)
diff --git a/parquet/src/arrow/array_reader/byte_array_dictionary.rs
b/parquet/src/arrow/array_reader/byte_array_dictionary.rs
index 14a2bd1241..f4d84503bb 100644
--- a/parquet/src/arrow/array_reader/byte_array_dictionary.rs
+++ b/parquet/src/arrow/array_reader/byte_array_dictionary.rs
@@ -31,7 +31,7 @@ use crate::arrow::schema::parquet_to_arrow_field;
use crate::basic::{ConvertedType, Encoding};
use crate::column::page::PageIterator;
use crate::column::reader::decoder::ColumnValueDecoder;
-use crate::encodings::rle::RleDecoder;
+use crate::encodings::rle::{MAX_RLE_DICTIONARY_BIT_WIDTH, RleDecoder};
use crate::errors::{ParquetError, Result};
use crate::schema::types::ColumnDescPtr;
use crate::util::bit_util::FromBitpacked;
@@ -298,7 +298,14 @@ where
) -> Result<()> {
let decoder = match encoding {
Encoding::RLE_DICTIONARY | Encoding::PLAIN_DICTIONARY => {
- let bit_width = data[0];
+ let bit_width = *data
+ .first()
+ .ok_or_else(|| general_err!("dictionary index page is
empty"))?;
+ if bit_width > MAX_RLE_DICTIONARY_BIT_WIDTH {
+ return Err(general_err!(
+ "Invalid or corrupted RLE bit width {bit_width}. Max
allowed is {MAX_RLE_DICTIONARY_BIT_WIDTH}"
+ ));
+ }
let mut decoder = RleDecoder::new(bit_width);
decoder.set_data(data.slice(1..))?;
MaybeDictionaryDecoder::Dict {
@@ -688,4 +695,31 @@ mod tests {
assert_eq!(array.logical_null_count(), 8);
}
}
+
+ #[test]
+ fn test_dictionary_decoder_empty_data() {
+ let column_desc = utf8_column();
+ let mut decoder = DictionaryDecoder::<i32, i32>::new(&column_desc);
+ let err = decoder
+ .set_data(Encoding::RLE_DICTIONARY, Bytes::new(), 0, None)
+ .unwrap_err();
+ assert_eq!(
+ err.to_string(),
+ "Parquet error: dictionary index page is empty"
+ );
+ }
+
+ #[test]
+ fn test_dictionary_decoder_invalid_bit_width() {
+ let column_desc = utf8_column();
+ let mut decoder = DictionaryDecoder::<i32, i32>::new(&column_desc);
+ let data = Bytes::from_static(&[33, 0, 0, 0]);
+ let err = decoder
+ .set_data(Encoding::RLE_DICTIONARY, data, 1, None)
+ .unwrap_err();
+ assert_eq!(
+ err.to_string(),
+ "Parquet error: Invalid or corrupted RLE bit width 33. Max allowed
is 32"
+ );
+ }
}
diff --git a/parquet/src/arrow/decoder/dictionary_index.rs
b/parquet/src/arrow/decoder/dictionary_index.rs
index 7a4b77f89d..6640fa7381 100644
--- a/parquet/src/arrow/decoder/dictionary_index.rs
+++ b/parquet/src/arrow/decoder/dictionary_index.rs
@@ -17,8 +17,8 @@
use bytes::Bytes;
-use crate::encodings::rle::RleDecoder;
-use crate::errors::Result;
+use crate::encodings::rle::{MAX_RLE_DICTIONARY_BIT_WIDTH, RleDecoder};
+use crate::errors::{ParquetError, Result};
/// Decoder for `Encoding::RLE_DICTIONARY` indices
pub struct DictIndexDecoder {
@@ -43,7 +43,14 @@ impl DictIndexDecoder {
/// Create a new [`DictIndexDecoder`] with the provided data page, the
number of levels
/// associated with this data page, and the number of non-null values (if
known)
pub fn new(data: Bytes, num_levels: usize, num_values: Option<usize>) ->
Result<Self> {
- let bit_width = data[0];
+ let bit_width = *data
+ .first()
+ .ok_or_else(|| general_err!("dictionary index page is empty"))?;
+ if bit_width > MAX_RLE_DICTIONARY_BIT_WIDTH {
+ return Err(general_err!(
+ "Invalid or corrupted RLE bit width {bit_width}. Max allowed
is {MAX_RLE_DICTIONARY_BIT_WIDTH}"
+ ));
+ }
let mut decoder = RleDecoder::new(bit_width);
decoder.set_data(data.slice(1..))?;
@@ -119,3 +126,31 @@ impl DictIndexDecoder {
Ok(values_skip)
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_dict_index_decoder_empty_data() {
+ let Err(err) = DictIndexDecoder::new(Bytes::new(), 0, None) else {
+ panic!("expected error");
+ };
+ assert_eq!(
+ err.to_string(),
+ "Parquet error: dictionary index page is empty"
+ );
+ }
+
+ #[test]
+ fn test_dict_index_decoder_invalid_bit_width() {
+ let data = Bytes::from_static(&[33, 0, 0, 0]);
+ let Err(err) = DictIndexDecoder::new(data, 1, None) else {
+ panic!("expected error");
+ };
+ assert_eq!(
+ err.to_string(),
+ "Parquet error: Invalid or corrupted RLE bit width 33. Max allowed
is 32"
+ );
+ }
+}
diff --git a/parquet/src/encodings/decoding.rs
b/parquet/src/encodings/decoding.rs
index 85f3bd495b..2b2bae302e 100644
--- a/parquet/src/encodings/decoding.rs
+++ b/parquet/src/encodings/decoding.rs
@@ -21,7 +21,7 @@ use bytes::Bytes;
use num_traits::{FromPrimitive, WrappingAdd};
use std::{cmp, marker::PhantomData, mem};
-use super::rle::RleDecoder;
+use super::rle::{MAX_RLE_DICTIONARY_BIT_WIDTH, RleDecoder};
use crate::basic::*;
use crate::data_type::private::ParquetValueType;
@@ -386,10 +386,9 @@ impl<T: DataType> Decoder<T> for DictDecoder<T> {
}
let bit_width = data.as_ref()[0];
- if bit_width > 32 {
+ if bit_width > MAX_RLE_DICTIONARY_BIT_WIDTH {
return Err(general_err!(
- "Invalid or corrupted RLE bit width {}. Max allowed is 32",
- bit_width
+ "Invalid or corrupted RLE bit width {bit_width}. Max allowed
is {MAX_RLE_DICTIONARY_BIT_WIDTH}"
));
}
let mut rle_decoder = RleDecoder::new(bit_width);
diff --git a/parquet/src/encodings/rle.rs b/parquet/src/encodings/rle.rs
index a1994a4fe0..95538dfb41 100644
--- a/parquet/src/encodings/rle.rs
+++ b/parquet/src/encodings/rle.rs
@@ -41,6 +41,12 @@ use bytes::Bytes;
use crate::errors::{ParquetError, Result};
use crate::util::bit_util::{self, BitReader, BitWriter, FromBitpacked};
+/// Maximum bit width for dictionary page indices encoded with RLE /
Bit-Packing hybrid encoding.
+///
+/// Parquet dictionary indices are represented as 32-bit signed integers, so
dictionary
+/// index bit widths must not exceed 32.
+pub const MAX_RLE_DICTIONARY_BIT_WIDTH: u8 = 32;
+
/// Number of values in one bit-packed group. The Parquet RLE/bit-packing
hybrid
/// format always bit-packs values in multiples of this count (see the
/// [format
spec](https://github.com/apache/parquet-format/blob/master/Encodings.md#run-length-encoding--bit-packing-hybrid-rle--3):
@@ -360,7 +366,12 @@ pub struct RleDecoder {
}
impl RleDecoder {
+ /// Creates a new `RleDecoder` with the specified bit width.
+ ///
+ /// Bit width must be between 0 and 64 (inclusive). Note that for
dictionary indices
+ /// specifically, the bit width cannot exceed
[`MAX_RLE_DICTIONARY_BIT_WIDTH`] (32).
pub fn new(bit_width: u8) -> Self {
+ debug_assert!(bit_width <= 64, "Bit width must be <= 64");
RleDecoder {
bit_width,
rle_left: 0,
