This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-11481-ecb0172a25de747ad4f055031efa98caac0490e7 in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
commit 418340d5188f6e2949c4a0545b3e437dfdc848f8 Author: Jörn Horstmann <[email protected]> AuthorDate: Thu Oct 15 10:51:09 2026 +0000 Make ByteStreamSplit and Alp Encoder/Decoder and their constructors experimentally public (#11481) # Which issue does this PR close? <!-- It is not necessary to raise an issue, but it is encouraged if it is a non-trivial enhancement/bug. If this PR relates to an existing issue, but does not fully resolve it, please change the wording below from "Closes #NNN" to "Relates to #NNN" or "Part of #NNN" to avoid false closing an issue. --> - Closes #11480 # Rationale for this change I would like to reuse some of the lower-level parquet implementation outside of this crate, and also benchmark against different Decoder implementations. All other encoders and decoders were already public, so I would assume this was an oversight for BSS and ALP when they were added. <!-- Why are you proposing this change? If this is already explained clearly in the issue then this section is not needed. Explaining clearly why changes are proposed helps reviewers understand your changes and offer better suggestions for fixes. Please try to explain the problem you are trying to solve in terms of the user-visible behavior, rather than the implementation. --> # What changes are included in this PR? Publicly exporting these encoders and decoders and making their constructors and other required traits public. <!-- There is no need to duplicate the description in the issue here but it is sometimes worth providing a summary of the individual changes in this PR. --> # Are these changes tested? Verified locally that they are publicly accessible without warnings about visibility. # Are there any significant user-facing changes? No --------- Co-authored-by: Andrew Lamb <[email protected]> --- parquet/src/encodings/alp.rs | 8 ++------ parquet/src/encodings/decoding.rs | 4 ++-- parquet/src/encodings/decoding/alp_decoder.rs | 14 ++++++++++++-- .../src/encodings/decoding/byte_stream_split_decoder.rs | 10 ++++++++-- parquet/src/encodings/encoding/alp_encoder.rs | 13 +++++++++++-- .../src/encodings/encoding/byte_stream_split_encoder.rs | 10 ++++++++-- parquet/src/encodings/encoding/mod.rs | 4 ++-- 7 files changed, 45 insertions(+), 18 deletions(-) diff --git a/parquet/src/encodings/alp.rs b/parquet/src/encodings/alp.rs index 8c51039a2a..a3d08f7039 100644 --- a/parquet/src/encodings/alp.rs +++ b/parquet/src/encodings/alp.rs @@ -245,9 +245,7 @@ impl<Exact: AlpExact> ForInfo<Exact> { /// unsigned wrapping arithmetic: this avoids signed overflow when a vector's /// range exceeds the signed maximum, and unpacking needs no sign extension. /// Signed interpretation is applied later during decimal reconstruction. -pub(crate) trait AlpExact: - Copy + std::fmt::Debug + PartialEq + BitPacking + Default -{ +pub trait AlpExact: Copy + std::fmt::Debug + PartialEq + BitPacking + Default { const WIDTH: usize; type Signed: BitPacking + Copy + Ord + std::fmt::Debug + Send; fn from_le_slice(slice: &[u8]) -> Self; @@ -406,9 +404,7 @@ pub(crate) const ALP_NEG_POW10_F64: [f64; 19] = [ /// float side of the codec, the decimal scaling and rounding that turn floats /// into exact integers and back, while [`AlpExact`] owns the integer side, the /// FOR and bit-packing arithmetic on those encoded values. -pub(crate) trait AlpFloat: - Copy + Default + PartialEq + std::ops::Mul<Output = Self> -{ +pub trait AlpFloat: Copy + Default + PartialEq + std::ops::Mul<Output = Self> { type Exact: AlpExact + FromBytes; type Scale: Copy + Send; diff --git a/parquet/src/encodings/decoding.rs b/parquet/src/encodings/decoding.rs index 6346f205e4..ba93bce45e 100644 --- a/parquet/src/encodings/decoding.rs +++ b/parquet/src/encodings/decoding.rs @@ -26,8 +26,8 @@ use super::rle::{MAX_RLE_DICTIONARY_BIT_WIDTH, RleDecoder}; use crate::basic::*; use crate::data_type::private::ParquetValueType; use crate::data_type::*; -use crate::encodings::decoding::alp_decoder::AlpDecoder; -use crate::encodings::decoding::byte_stream_split_decoder::{ +pub use crate::encodings::decoding::alp_decoder::AlpDecoder; +pub use crate::encodings::decoding::byte_stream_split_decoder::{ ByteStreamSplitDecoder, VariableWidthByteStreamSplitDecoder, }; use crate::errors::{ParquetError, Result}; diff --git a/parquet/src/encodings/decoding/alp_decoder.rs b/parquet/src/encodings/decoding/alp_decoder.rs index 47f5edb4d3..887aafc0b2 100644 --- a/parquet/src/encodings/decoding/alp_decoder.rs +++ b/parquet/src/encodings/decoding/alp_decoder.rs @@ -362,7 +362,7 @@ fn decode_range<Value: AlpFloat>( /// Values are decoded directly into the caller's output buffer, one vector at a /// time: `set_data` validates the page header, and each vector is parsed and /// decoded on demand as the read cursor reaches it. -pub(crate) struct AlpDecoder<T: DataType> +pub struct AlpDecoder<T: DataType> where T::T: AlpFloat, <T::T as AlpFloat>::Exact: Send, @@ -385,12 +385,22 @@ where num_values: usize, } +impl<T: DataType> Default for AlpDecoder<T> +where + T::T: AlpFloat, + <T::T as AlpFloat>::Exact: Send, +{ + fn default() -> Self { + Self::new() + } +} + impl<T: DataType> AlpDecoder<T> where T::T: AlpFloat, <T::T as AlpFloat>::Exact: Send, { - pub(crate) fn new() -> Self { + pub fn new() -> Self { Self { // Benign placeholder header; replaced on the first `set_data`. The // cursor never consults it while `num_values == 0`. diff --git a/parquet/src/encodings/decoding/byte_stream_split_decoder.rs b/parquet/src/encodings/decoding/byte_stream_split_decoder.rs index 2e26e4d3aa..477c5106df 100644 --- a/parquet/src/encodings/decoding/byte_stream_split_decoder.rs +++ b/parquet/src/encodings/decoding/byte_stream_split_decoder.rs @@ -33,8 +33,14 @@ pub struct ByteStreamSplitDecoder<T: DataType> { values_decoded: usize, } +impl<T: DataType> Default for ByteStreamSplitDecoder<T> { + fn default() -> Self { + Self::new() + } +} + impl<T: DataType> ByteStreamSplitDecoder<T> { - pub(crate) fn new() -> Self { + pub fn new() -> Self { Self { _phantom: PhantomData, encoded_bytes: Bytes::new(), @@ -146,7 +152,7 @@ pub struct VariableWidthByteStreamSplitDecoder<T: DataType> { } impl<T: DataType> VariableWidthByteStreamSplitDecoder<T> { - pub(crate) fn new(type_length: i32) -> Self { + pub fn new(type_length: i32) -> Self { Self { _phantom: PhantomData, encoded_bytes: Bytes::new(), diff --git a/parquet/src/encodings/encoding/alp_encoder.rs b/parquet/src/encodings/encoding/alp_encoder.rs index 9b2629d410..95c7858522 100644 --- a/parquet/src/encodings/encoding/alp_encoder.rs +++ b/parquet/src/encodings/encoding/alp_encoder.rs @@ -601,7 +601,7 @@ impl<F: AlpFloat> StreamingPage<F> { /// The first page is buffered whole: ALP samples it to choose the column chunk's /// candidate `(exponent, factor)` set, which needs all of the page's data. Once /// that preset is fixed, later pages are encoded incrementally, a vector at a -/// time, without ever holding the whole page of raw floats (see [`StreamingPage`]). +/// time, without ever holding the whole page of raw floats. pub struct AlpEncoder<T: DataType> where T::T: AlpFloat, @@ -618,11 +618,20 @@ where streaming: StreamingPage<T::T>, } +impl<T: DataType> Default for AlpEncoder<T> +where + T::T: AlpFloat, +{ + fn default() -> Self { + Self::new() + } +} + impl<T: DataType> AlpEncoder<T> where T::T: AlpFloat, { - pub(crate) fn new() -> Self { + pub fn new() -> Self { Self { values: Vec::new(), preset: None, diff --git a/parquet/src/encodings/encoding/byte_stream_split_encoder.rs b/parquet/src/encodings/encoding/byte_stream_split_encoder.rs index 9486e13da5..0f9e021cba 100644 --- a/parquet/src/encodings/encoding/byte_stream_split_encoder.rs +++ b/parquet/src/encodings/encoding/byte_stream_split_encoder.rs @@ -31,8 +31,14 @@ pub struct ByteStreamSplitEncoder<T> { _p: PhantomData<T>, } +impl<T: DataType> Default for ByteStreamSplitEncoder<T> { + fn default() -> Self { + Self::new() + } +} + impl<T: DataType> ByteStreamSplitEncoder<T> { - pub(crate) fn new() -> Self { + pub fn new() -> Self { Self { buffer: Vec::new(), _p: PhantomData, @@ -119,7 +125,7 @@ pub struct VariableWidthByteStreamSplitEncoder<T> { } impl<T: DataType> VariableWidthByteStreamSplitEncoder<T> { - pub(crate) fn new(type_length: i32) -> Self { + pub fn new(type_length: i32) -> Self { Self { buffer: Vec::new(), type_width: type_length as usize, diff --git a/parquet/src/encodings/encoding/mod.rs b/parquet/src/encodings/encoding/mod.rs index 4f50cd95cc..40de8346eb 100644 --- a/parquet/src/encodings/encoding/mod.rs +++ b/parquet/src/encodings/encoding/mod.rs @@ -29,8 +29,8 @@ use crate::schema::types::ColumnDescPtr; use crate::util::bit_util::{BitWriter, num_required_bits}; use crate::util::prefix::common_prefix_length; -use alp_encoder::AlpEncoder; -use byte_stream_split_encoder::{ByteStreamSplitEncoder, VariableWidthByteStreamSplitEncoder}; +pub use alp_encoder::AlpEncoder; +pub use byte_stream_split_encoder::{ByteStreamSplitEncoder, VariableWidthByteStreamSplitEncoder}; use bytes::Bytes; pub use dict_encoder::DictEncoder;
