prtkgaur commented on code in PR #48345:
URL: https://github.com/apache/arrow/pull/48345#discussion_r3920277220
##########
cpp/src/parquet/encoding_benchmark.cc:
##########
@@ -661,6 +661,78 @@
BENCHMARK(BM_ByteStreamSplitEncode_Float_Neon)->Apply(ByteStreamSplitApply);
BENCHMARK(BM_ByteStreamSplitEncode_Double_Neon)->Apply(ByteStreamSplitApply);
#endif
+// ----------------------------------------------------------------------
+// ALP encoding/decoding benchmarks
+
+static void BM_AlpEncodingFloat(benchmark::State& state) {
+ std::vector<float> values(state.range(0), 64.0f);
+ auto encoder = MakeTypedEncoder<FloatType>(Encoding::ALP);
+ for (auto _ : state) {
+ encoder->Put(values.data(), static_cast<int>(values.size()));
+ encoder->FlushValues();
+ }
+ state.SetBytesProcessed(state.iterations() * state.range(0) * sizeof(float));
+ state.SetItemsProcessed(state.iterations() * state.range(0));
+}
+
+BENCHMARK(BM_AlpEncodingFloat)->Range(MIN_RANGE, MAX_RANGE);
+
+static void BM_AlpDecodingFloat(benchmark::State& state) {
+ std::vector<float> values(state.range(0), 64.0f);
+ auto encoder = MakeTypedEncoder<FloatType>(Encoding::ALP);
+ encoder->Put(values.data(), static_cast<int>(values.size()));
+ std::shared_ptr<Buffer> buf = encoder->FlushValues();
+
+ for (auto _ : state) {
+ auto decoder = MakeTypedDecoder<FloatType>(Encoding::ALP);
+ decoder->SetData(static_cast<int>(values.size()), buf->data(),
+ static_cast<int>(buf->size()));
+ std::vector<float> output(values.size());
+ decoder->Decode(output.data(), static_cast<int>(values.size()));
+ benchmark::ClobberMemory();
+ }
+ state.SetBytesProcessed(state.iterations() * state.range(0) * sizeof(float));
+ state.SetItemsProcessed(state.iterations() * state.range(0));
+}
+
+BENCHMARK(BM_AlpDecodingFloat)->Range(MIN_RANGE, MAX_RANGE);
+
+static void BM_AlpEncodingDouble(benchmark::State& state) {
+ std::vector<double> values(state.range(0), 64.0);
+ auto encoder = MakeTypedEncoder<DoubleType>(Encoding::ALP);
+ for (auto _ : state) {
+ encoder->Put(values.data(), static_cast<int>(values.size()));
+ encoder->FlushValues();
+ }
+ state.SetBytesProcessed(state.iterations() * state.range(0) *
sizeof(double));
+ state.SetItemsProcessed(state.iterations() * state.range(0));
+}
+
+BENCHMARK(BM_AlpEncodingDouble)->Range(MIN_RANGE, MAX_RANGE);
+
+static void BM_AlpDecodingDouble(benchmark::State& state) {
+ std::vector<double> values(state.range(0), 64.0);
+ auto encoder = MakeTypedEncoder<DoubleType>(Encoding::ALP);
+ encoder->Put(values.data(), static_cast<int>(values.size()));
+ std::shared_ptr<Buffer> buf = encoder->FlushValues();
+
+ for (auto _ : state) {
+ auto decoder = MakeTypedDecoder<DoubleType>(Encoding::ALP);
Review Comment:
To be exact: the float one was still allocating inside the loop and timing
`SetData` when I posted that. Fixed in 3b1b6e1, so both match now.
##########
cpp/src/parquet/encoder.cc:
##########
@@ -997,6 +999,104 @@ class ByteStreamSplitEncoder<FLBAType> : public
ByteStreamSplitEncoderBase<FLBAT
}
};
+// ----------------------------------------------------------------------
+// ALP encoder (Adaptive Lossless floating-Point)
+
+template <typename DType>
+class AlpEncoder : public EncoderImpl, virtual public TypedEncoder<DType> {
+ public:
+ using T = typename DType::c_type;
+ using ArrowType = typename EncodingTraits<DType>::ArrowType;
+ using TypedEncoder<DType>::Put;
+
+ explicit AlpEncoder(const ColumnDescriptor* descr,
+ ::arrow::MemoryPool* pool =
::arrow::default_memory_pool(),
+ int32_t vector_size =
::arrow::util::alp::AlpConstants::kAlpVectorSize)
+ : EncoderImpl(descr, Encoding::ALP, pool),
+ sink_{pool},
+ vector_size_(vector_size) {
+ static_assert(std::is_same<T, float>::value || std::is_same<T,
double>::value,
+ "ALP only supports float and double types");
+ if (vector_size_ <= 0 || (vector_size_ & (vector_size_ - 1)) != 0) {
Review Comment:
`std::has_single_bit` covers it, so nothing needed adding to `bit_util` —
Arrow mandates C++20, so `<bit>` is available unconditionally.
The check has also moved out of the encoder. `vector_size` isn't
configurable there any more (it's a `static constexpr`, since a reader takes
the vector size from the page header), so there's nothing left to validate at
this site; `ValidateVectorSize` in the codec does it where the value can
actually vary. The other raw expression, the bits-to-bytes rounding, is
`bit_util::BytesForBits` throughout.
##########
cpp/src/parquet/encoder.cc:
##########
@@ -997,6 +999,104 @@ class ByteStreamSplitEncoder<FLBAType> : public
ByteStreamSplitEncoderBase<FLBAT
}
};
+// ----------------------------------------------------------------------
+// ALP encoder (Adaptive Lossless floating-Point)
+
+template <typename DType>
+class AlpEncoder : public EncoderImpl, virtual public TypedEncoder<DType> {
+ public:
+ using T = typename DType::c_type;
+ using ArrowType = typename EncodingTraits<DType>::ArrowType;
+ using TypedEncoder<DType>::Put;
+
+ explicit AlpEncoder(const ColumnDescriptor* descr,
+ ::arrow::MemoryPool* pool =
::arrow::default_memory_pool(),
+ int32_t vector_size =
::arrow::util::alp::AlpConstants::kAlpVectorSize)
+ : EncoderImpl(descr, Encoding::ALP, pool),
+ sink_{pool},
+ vector_size_(vector_size) {
+ static_assert(std::is_same<T, float>::value || std::is_same<T,
double>::value,
+ "ALP only supports float and double types");
+ if (vector_size_ <= 0 || (vector_size_ & (vector_size_ - 1)) != 0) {
+ throw ParquetException("ALP vector_size must be a positive power of 2,
got " +
+ std::to_string(vector_size_));
+ }
+ if (vector_size_ > (1 <<
::arrow::util::alp::AlpConstants::kMaxLogVectorSize)) {
+ throw ParquetException("ALP vector_size exceeds maximum " +
+ std::to_string(1 <<
::arrow::util::alp::AlpConstants::kMaxLogVectorSize) +
+ ", got " + std::to_string(vector_size_));
+ }
+ }
+
+ int64_t EstimatedDataEncodedSize() override { return sink_.length(); }
+
+ std::shared_ptr<Buffer> FlushValues() override {
+ if (sink_.length() == 0) {
+ // Empty buffer case
+ PARQUET_ASSIGN_OR_THROW(auto buf, sink_.Finish());
+ return buf;
+ }
+
+ // Call AlpCodec::Encode() - it handles sampling, preset selection, and
compression
+ const int64_t numElements = sink_.length() /
static_cast<int64_t>(sizeof(T));
+ int64_t compSize =
Review Comment:
Applied — it's `PARQUET_ASSIGN_OR_THROW(int64_t comp_size, ...)` now, since
`GetMaxCompressedSize` returns `Result` after your other comment.
##########
cpp/src/parquet/decoder.cc:
##########
@@ -2372,6 +2327,125 @@ class ByteStreamSplitDecoder<FLBAType> : public
ByteStreamSplitDecoderBase<FLBAT
}
};
+// ----------------------------------------------------------------------
+// ALP decoder (Adaptive Lossless floating-Point)
+
+template <typename DType>
+class AlpDecoder : public TypedDecoderImpl<DType> {
+ public:
+ using Base = TypedDecoderImpl<DType>;
+ using T = typename DType::c_type;
+
+ explicit AlpDecoder(const ColumnDescriptor* descr)
+ : Base(descr, Encoding::ALP), current_offset_{0}, needs_decode_{false} {
+ static_assert(std::is_same<T, float>::value || std::is_same<T,
double>::value,
+ "ALP only supports float and double types");
+ }
+
+ void SetData(int num_values, const uint8_t* data, int len) final {
+ Base::SetData(num_values, data, len);
+ current_offset_ = 0;
+ if (num_values > 0 && len <= 0) {
+ throw ParquetException("ALP SetData: num_values=" +
std::to_string(num_values) +
+ " but len=" + std::to_string(len));
+ }
+ needs_decode_ = (num_values > 0);
+ decoded_buffer_.clear();
+ }
+
+ int Decode(T* buffer, int max_values) override {
+ // Fast path: decode directly into output buffer if requesting all values
+ if (needs_decode_ && max_values >= this->num_values_) {
+ PARQUET_THROW_NOT_OK(::arrow::util::alp::AlpCodec<T>::Decode(
+ this->num_values_, this->data_, this->len_,
+ buffer));
+
+ const int decoded = this->num_values_;
+ this->num_values_ = 0;
+ needs_decode_ = false;
+ return decoded;
+ }
+
+ // Slow path: partial read - decode to intermediate buffer
+ // ALP Bit unpacker needs batches of 64
+ if (needs_decode_) {
+ decoded_buffer_.resize(this->num_values_);
+ PARQUET_THROW_NOT_OK(::arrow::util::alp::AlpCodec<T>::Decode(
+ this->num_values_, this->data_, this->len_,
+ decoded_buffer_.data()));
+ needs_decode_ = false;
+ }
+
+ // Copy from intermediate buffer
+ const int values_to_decode = std::min(
+ max_values,
+ static_cast<int>(decoded_buffer_.size() - current_offset_));
+
+ if (values_to_decode > 0) {
+ std::memcpy(buffer, decoded_buffer_.data() + current_offset_,
+ values_to_decode * sizeof(T));
+ current_offset_ += values_to_decode;
+ this->num_values_ -= values_to_decode;
+ }
+
+ return values_to_decode;
+ }
+
+ int DecodeArrow(int num_values, int null_count, const uint8_t* valid_bits,
+ int64_t valid_bits_offset,
+ typename EncodingTraits<DType>::Accumulator* builder)
override {
+ const int values_to_decode = num_values - null_count;
+ if (ARROW_PREDICT_FALSE(this->num_values_ < values_to_decode)) {
+ ParquetException::EofException("ALP DecodeArrow: Not enough values
available. "
Review Comment:
It throws — `throw ParquetException(...)`, not a bare construction. Same for
the surrounding paths, which go through `PARQUET_THROW_NOT_OK`.
##########
cpp/src/arrow/util/alp/alp_sampler.h:
##########
@@ -0,0 +1,123 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// ALP sampler for collecting samples and creating encoding presets
+
+#pragma once
+
+#include <optional>
+#include <vector>
+
+#include "arrow/util/alp/alp.h"
+#include "arrow/util/span.h"
+
+namespace arrow {
+namespace util {
+namespace alp {
+
+// ----------------------------------------------------------------------
+// AlpSampler
+
+/// \class AlpSampler
+/// \brief Collects samples from data to be compressed with ALP
+///
+/// Usage: Call AddSample() or AddSampleVector() multiple times to collect
+/// samples, then call Finalize() to retrieve the resulting preset.
+///
+/// \tparam T the floating point type (float or double) to sample
+template <typename T>
+class AlpSampler {
+ public:
+ /// \brief Default constructor
+ AlpSampler();
+
+ /// \brief Helper struct containing the preset for ALP compression
+ struct AlpSamplerResult {
+ AlpEncodingParameters alp_parameters;
+ };
+
+ /// \brief Add a sample of arbitrary size
+ ///
+ /// The sample is internally separated into vectors on which
AddSampleVector()
+ /// is called.
+ ///
+ /// \param[in] input the input data to sample from
+ void AddSample(arrow::util::span<const T> input);
+
+ /// \brief Add a single vector as a sample
+ ///
+ /// \param[in] input the input vector to add.
+ /// Size should be <= AlpConstants::kAlpVectorSize.
+ void AddSampleVector(arrow::util::span<const T> input);
+
+ /// \brief Finalize sampling and generate the encoding preset
+ ///
+ /// \return an AlpSamplerResult containing the generated encoding preset
+ AlpSamplerResult Finalize();
+
+ private:
+ /// \brief Helper struct to encapsulate settings used for sampling
+ struct AlpSamplingParameters {
+ uint64_t num_lookup_value;
+ uint64_t num_sampled_increments;
+ uint64_t num_sampled_values;
+ };
+
+ /// \brief Calculate sampling parameters for the current vector
+ ///
+ /// \param[in] num_current_vector_values number of values in current vector
+ /// \return the sampling parameters to use
+ AlpSamplingParameters GetAlpSamplingParameters(uint64_t
num_current_vector_values);
+
+ /// \brief Check if the current vector must be ignored for sampling
+ ///
+ /// \param[in] vectors_count the total number of vectors processed so far
+ /// \param[in] vectors_sampled_count the number of vectors sampled so far
+ /// \param[in] num_current_vector_values number of values in current vector
+ /// \return true if the current vector should be skipped, false otherwise
+ bool MustSkipSamplingFromCurrentVector(uint64_t vectors_count,
+ uint64_t vectors_sampled_count,
+ uint64_t num_current_vector_values);
+
+ /// Count of vectors that have been sampled
+ uint64_t vectors_sampled_count_ = 0;
Review Comment:
Done — the counters and the `MustSkipSamplingFromCurrentVector` parameters
are all `int64_t` now, no `uint64_t` left in the sampler.
##########
cpp/src/parquet/encoding_alp_benchmark.cc:
##########
@@ -0,0 +1,1824 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <chrono>
+#include <cmath>
+#include <cstring>
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+#include <memory>
+#include <random>
+#include <sstream>
+#include <unistd.h>
+#include <unordered_set>
+#include <vector>
+
+#include <benchmark/benchmark.h>
+
+#include "arrow/buffer.h"
+#include "arrow/util/alp/alp_codec.h"
+#include "arrow/util/compression.h"
+#include "parquet/encoding.h"
+#include "parquet/schema.h"
+#include "parquet/types.h"
+
+// This file benchmarks multiple encoding schemes for floating point values in
+// Parquet. Structure mirrors Snowflake's FloatComprBenchmark.cpp
Review Comment:
Removed. The ALP benchmarks live in `encoding_benchmark.cc` now rather than
in their own file, and no Snowflake reference remains anywhere under `cpp/`.
##########
cpp/src/parquet/encoder.cc:
##########
@@ -997,6 +999,104 @@ class ByteStreamSplitEncoder<FLBAType> : public
ByteStreamSplitEncoderBase<FLBAT
}
};
+// ----------------------------------------------------------------------
+// ALP encoder (Adaptive Lossless floating-Point)
+
+template <typename DType>
+class AlpEncoder : public EncoderImpl, virtual public TypedEncoder<DType> {
+ public:
+ using T = typename DType::c_type;
+ using ArrowType = typename EncodingTraits<DType>::ArrowType;
+ using TypedEncoder<DType>::Put;
+
+ explicit AlpEncoder(const ColumnDescriptor* descr,
+ ::arrow::MemoryPool* pool =
::arrow::default_memory_pool(),
+ int32_t vector_size =
::arrow::util::alp::AlpConstants::kAlpVectorSize)
+ : EncoderImpl(descr, Encoding::ALP, pool),
+ sink_{pool},
+ vector_size_(vector_size) {
+ static_assert(std::is_same<T, float>::value || std::is_same<T,
double>::value,
+ "ALP only supports float and double types");
+ if (vector_size_ <= 0 || (vector_size_ & (vector_size_ - 1)) != 0) {
+ throw ParquetException("ALP vector_size must be a positive power of 2,
got " +
+ std::to_string(vector_size_));
+ }
+ if (vector_size_ > (1 <<
::arrow::util::alp::AlpConstants::kMaxLogVectorSize)) {
+ throw ParquetException("ALP vector_size exceeds maximum " +
+ std::to_string(1 <<
::arrow::util::alp::AlpConstants::kMaxLogVectorSize) +
+ ", got " + std::to_string(vector_size_));
+ }
+ }
+
+ int64_t EstimatedDataEncodedSize() override { return sink_.length(); }
+
+ std::shared_ptr<Buffer> FlushValues() override {
+ if (sink_.length() == 0) {
+ // Empty buffer case
+ PARQUET_ASSIGN_OR_THROW(auto buf, sink_.Finish());
+ return buf;
+ }
+
+ // Call AlpCodec::Encode() - it handles sampling, preset selection, and
compression
+ const int64_t numElements = sink_.length() /
static_cast<int64_t>(sizeof(T));
Review Comment:
Applied as written.
##########
cpp/src/parquet/encoding_test.cc:
##########
@@ -2660,4 +2595,407 @@ TEST(DeltaByteArrayEncodingAdHoc, ArrowDirectPut) {
}
}
+// ----------------------------------------------------------------------
+// ALP encoding tests for float/double
+
+template <typename Type>
+class TestAlpEncoding : public TestEncodingBase<Type> {
+ public:
+ using c_type = typename Type::c_type;
+ static constexpr int TYPE = Type::type_num;
+ static constexpr size_t kNumRoundTrips = 3;
+
+ void CheckRoundtrip() override {
+ auto encoder =
+ MakeTypedEncoder<Type>(Encoding::ALP, /*use_dictionary=*/false,
descr_.get());
+ auto decoder = MakeTypedDecoder<Type>(Encoding::ALP, descr_.get());
+
+ for (size_t i = 0; i < kNumRoundTrips; ++i) {
+ encoder->Put(draws_, num_values_);
+ encode_buffer_ = encoder->FlushValues();
+
+ decoder->SetData(num_values_, encode_buffer_->data(),
+ static_cast<int>(encode_buffer_->size()));
+ int values_decoded = decoder->Decode(decode_buf_, num_values_);
+ ASSERT_EQ(num_values_, values_decoded);
+
+ // Use memcmp for bit-exact comparison (important for -0.0, NaN bit
patterns)
+ ASSERT_EQ(0, std::memcmp(draws_, decode_buf_, num_values_ *
sizeof(c_type)));
+ }
+ }
+
+ void CheckRoundtripSpaced(const uint8_t* valid_bits,
+ int64_t valid_bits_offset) override {
+ auto encoder =
+ MakeTypedEncoder<Type>(Encoding::ALP, /*use_dictionary=*/false,
descr_.get());
+ auto decoder = MakeTypedDecoder<Type>(Encoding::ALP, descr_.get());
+
+ int null_count = 0;
+ for (auto i = 0; i < num_values_; i++) {
+ if (!bit_util::GetBit(valid_bits, valid_bits_offset + i)) {
+ null_count++;
+ }
+ }
+
+ for (size_t i = 0; i < kNumRoundTrips; ++i) {
+ encoder->PutSpaced(draws_, num_values_, valid_bits, valid_bits_offset);
+ encode_buffer_ = encoder->FlushValues();
+
+ decoder->SetData(num_values_ - null_count, encode_buffer_->data(),
+ static_cast<int>(encode_buffer_->size()));
+ auto values_decoded = decoder->DecodeSpaced(decode_buf_, num_values_,
null_count,
+ valid_bits,
valid_bits_offset);
+ ASSERT_EQ(num_values_, values_decoded);
+
+ // Verify only valid values
+ for (int j = 0; j < num_values_; ++j) {
+ if (bit_util::GetBit(valid_bits, valid_bits_offset + j)) {
+ ASSERT_EQ(0, std::memcmp(&draws_[j], &decode_buf_[j],
sizeof(c_type))) << j;
+ }
+ }
+ }
+ }
+
+ void InitDataWithSpecialValues(int nvalues, int repeats) {
+ num_values_ = nvalues * repeats;
+ this->input_bytes_.resize(num_values_ * sizeof(c_type));
+ this->output_bytes_.resize(num_values_ * sizeof(c_type));
+ draws_ = reinterpret_cast<c_type*>(this->input_bytes_.data());
+ decode_buf_ = reinterpret_cast<c_type*>(this->output_bytes_.data());
+
+ // Fill with mix of normal and special values
+ for (int i = 0; i < nvalues; ++i) {
+ if (i % 20 == 0) {
+ draws_[i] = std::numeric_limits<c_type>::quiet_NaN();
+ } else if (i % 20 == 5) {
+ draws_[i] = std::numeric_limits<c_type>::infinity();
+ } else if (i % 20 == 10) {
+ draws_[i] = -std::numeric_limits<c_type>::infinity();
+ } else if (i % 20 == 15) {
+ draws_[i] = static_cast<c_type>(-0.0);
+ } else {
+ draws_[i] = static_cast<c_type>(i) * static_cast<c_type>(0.123);
+ }
+ }
+
+ // Repeat pattern
+ for (int j = 1; j < repeats; ++j) {
+ for (int i = 0; i < nvalues; ++i) {
+ draws_[nvalues * j + i] = draws_[i];
+ }
+ }
+ }
+
+ void InitDataDecimalPattern(int nvalues, int repeats) {
+ num_values_ = nvalues * repeats;
+ this->input_bytes_.resize(num_values_ * sizeof(c_type));
+ this->output_bytes_.resize(num_values_ * sizeof(c_type));
+ draws_ = reinterpret_cast<c_type*>(this->input_bytes_.data());
+ decode_buf_ = reinterpret_cast<c_type*>(this->output_bytes_.data());
+
+ // Decimal-like values that ALP compresses well
+ for (int i = 0; i < nvalues; ++i) {
+ draws_[i] = static_cast<c_type>(100.0 + i * 0.01);
+ }
+
+ for (int j = 1; j < repeats; ++j) {
+ for (int i = 0; i < nvalues; ++i) {
+ draws_[nvalues * j + i] = draws_[i];
+ }
+ }
+ }
+
+ void ExecuteSpecialValues(int nvalues, int repeats) {
+ InitDataWithSpecialValues(nvalues, repeats);
+ CheckRoundtrip();
+ }
+
+ void ExecuteDecimalPattern(int nvalues, int repeats) {
+ InitDataDecimalPattern(nvalues, repeats);
+ CheckRoundtrip();
+ }
+
+ protected:
+ USING_BASE_MEMBERS();
+};
+
+using AlpEncodedTypes = ::testing::Types<FloatType, DoubleType>;
+TYPED_TEST_SUITE(TestAlpEncoding, AlpEncodedTypes);
+
+TYPED_TEST(TestAlpEncoding, BasicRoundTrip) {
+ // Test various sizes including edge cases
+ for (int values = 1; values < 32; ++values) {
+ ASSERT_NO_FATAL_FAILURE(this->Execute(values, 1));
+ }
+
+ // Test exactly vector size (1024)
+ ASSERT_NO_FATAL_FAILURE(this->Execute(1024, 1));
+
+ // Test just under and over vector size
+ ASSERT_NO_FATAL_FAILURE(this->Execute(1023, 1));
+ ASSERT_NO_FATAL_FAILURE(this->Execute(1025, 1));
+
+ // Test multiple vectors
+ ASSERT_NO_FATAL_FAILURE(this->Execute(2048, 1));
+ ASSERT_NO_FATAL_FAILURE(this->Execute(3000, 1));
+}
+
+TYPED_TEST(TestAlpEncoding, RoundTripWithRepeats) {
+ // Test with repeated patterns
+ ASSERT_NO_FATAL_FAILURE(this->Execute(100, 10));
+ ASSERT_NO_FATAL_FAILURE(this->Execute(1024, 3));
+}
+
+TYPED_TEST(TestAlpEncoding, SpecialValues) {
+ // Test NaN, Inf, -Inf, -0.0 (these become exceptions in ALP)
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteSpecialValues(100, 1));
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteSpecialValues(1024, 1));
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteSpecialValues(2000, 1));
+}
+
+TYPED_TEST(TestAlpEncoding, DecimalPatterns) {
+ // Test decimal-like values that ALP compresses efficiently
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteDecimalPattern(100, 1));
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteDecimalPattern(1024, 1));
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteDecimalPattern(5000, 1));
+}
+
+TYPED_TEST(TestAlpEncoding, SpacedRoundTrip) {
+ // Test with null values at various probabilities
+ for (double null_prob : {0.0, 0.1, 0.5, 0.9}) {
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteSpaced(100, 1, 0, null_prob));
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteSpaced(1024, 1, 0, null_prob));
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteSpaced(2000, 1, 0, null_prob));
+ }
+
+ // Test with offset
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteSpaced(1024, 1, 7, 0.3));
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteSpaced(1024, 1, 64, 0.5));
+}
+
+TYPED_TEST(TestAlpEncoding, LargeDataset) {
+ // Test with large dataset (multiple pages worth)
+ ASSERT_NO_FATAL_FAILURE(this->Execute(100000, 1));
+}
+
+TYPED_TEST(TestAlpEncoding, RandomData) {
+ using c_type = typename TypeParam::c_type;
+ ::arrow::random::RandomArrayGenerator rag(42);
+
+ // Generate random float/double array
+ std::shared_ptr<::arrow::Array> arr;
+ if constexpr (std::is_same_v<c_type, float>) {
+ arr = rag.Float32(10000, -1000.0f, 1000.0f);
Review Comment:
Extended. There are now `SpecialValues` and `ConstantValues` in
`alp_test.cc`, plus `RandomData`, `AllExceptions` and `BoundaryValues` in
`encoding_test.cc`, all parameterized over float and double.
`RandomAndExtremes` pushes on the encodable/exception boundary specifically,
rather than only on well-behaved values.
##########
cpp/src/parquet/encoding_test.cc:
##########
@@ -2660,4 +2595,407 @@ TEST(DeltaByteArrayEncodingAdHoc, ArrowDirectPut) {
}
}
+// ----------------------------------------------------------------------
+// ALP encoding tests for float/double
+
+template <typename Type>
+class TestAlpEncoding : public TestEncodingBase<Type> {
+ public:
+ using c_type = typename Type::c_type;
+ static constexpr int TYPE = Type::type_num;
+ static constexpr size_t kNumRoundTrips = 3;
+
+ void CheckRoundtrip() override {
+ auto encoder =
+ MakeTypedEncoder<Type>(Encoding::ALP, /*use_dictionary=*/false,
descr_.get());
+ auto decoder = MakeTypedDecoder<Type>(Encoding::ALP, descr_.get());
+
+ for (size_t i = 0; i < kNumRoundTrips; ++i) {
+ encoder->Put(draws_, num_values_);
+ encode_buffer_ = encoder->FlushValues();
+
+ decoder->SetData(num_values_, encode_buffer_->data(),
+ static_cast<int>(encode_buffer_->size()));
+ int values_decoded = decoder->Decode(decode_buf_, num_values_);
+ ASSERT_EQ(num_values_, values_decoded);
+
+ // Use memcmp for bit-exact comparison (important for -0.0, NaN bit
patterns)
+ ASSERT_EQ(0, std::memcmp(draws_, decode_buf_, num_values_ *
sizeof(c_type)));
+ }
+ }
+
+ void CheckRoundtripSpaced(const uint8_t* valid_bits,
+ int64_t valid_bits_offset) override {
+ auto encoder =
+ MakeTypedEncoder<Type>(Encoding::ALP, /*use_dictionary=*/false,
descr_.get());
+ auto decoder = MakeTypedDecoder<Type>(Encoding::ALP, descr_.get());
+
+ int null_count = 0;
+ for (auto i = 0; i < num_values_; i++) {
+ if (!bit_util::GetBit(valid_bits, valid_bits_offset + i)) {
+ null_count++;
+ }
+ }
+
+ for (size_t i = 0; i < kNumRoundTrips; ++i) {
+ encoder->PutSpaced(draws_, num_values_, valid_bits, valid_bits_offset);
+ encode_buffer_ = encoder->FlushValues();
+
+ decoder->SetData(num_values_ - null_count, encode_buffer_->data(),
+ static_cast<int>(encode_buffer_->size()));
+ auto values_decoded = decoder->DecodeSpaced(decode_buf_, num_values_,
null_count,
+ valid_bits,
valid_bits_offset);
+ ASSERT_EQ(num_values_, values_decoded);
+
+ // Verify only valid values
+ for (int j = 0; j < num_values_; ++j) {
+ if (bit_util::GetBit(valid_bits, valid_bits_offset + j)) {
+ ASSERT_EQ(0, std::memcmp(&draws_[j], &decode_buf_[j],
sizeof(c_type))) << j;
+ }
+ }
+ }
+ }
+
+ void InitDataWithSpecialValues(int nvalues, int repeats) {
+ num_values_ = nvalues * repeats;
+ this->input_bytes_.resize(num_values_ * sizeof(c_type));
+ this->output_bytes_.resize(num_values_ * sizeof(c_type));
+ draws_ = reinterpret_cast<c_type*>(this->input_bytes_.data());
+ decode_buf_ = reinterpret_cast<c_type*>(this->output_bytes_.data());
+
+ // Fill with mix of normal and special values
+ for (int i = 0; i < nvalues; ++i) {
+ if (i % 20 == 0) {
+ draws_[i] = std::numeric_limits<c_type>::quiet_NaN();
+ } else if (i % 20 == 5) {
+ draws_[i] = std::numeric_limits<c_type>::infinity();
+ } else if (i % 20 == 10) {
+ draws_[i] = -std::numeric_limits<c_type>::infinity();
+ } else if (i % 20 == 15) {
+ draws_[i] = static_cast<c_type>(-0.0);
+ } else {
+ draws_[i] = static_cast<c_type>(i) * static_cast<c_type>(0.123);
+ }
+ }
+
+ // Repeat pattern
+ for (int j = 1; j < repeats; ++j) {
+ for (int i = 0; i < nvalues; ++i) {
+ draws_[nvalues * j + i] = draws_[i];
+ }
+ }
+ }
+
+ void InitDataDecimalPattern(int nvalues, int repeats) {
+ num_values_ = nvalues * repeats;
+ this->input_bytes_.resize(num_values_ * sizeof(c_type));
+ this->output_bytes_.resize(num_values_ * sizeof(c_type));
+ draws_ = reinterpret_cast<c_type*>(this->input_bytes_.data());
+ decode_buf_ = reinterpret_cast<c_type*>(this->output_bytes_.data());
+
+ // Decimal-like values that ALP compresses well
+ for (int i = 0; i < nvalues; ++i) {
+ draws_[i] = static_cast<c_type>(100.0 + i * 0.01);
+ }
+
+ for (int j = 1; j < repeats; ++j) {
+ for (int i = 0; i < nvalues; ++i) {
+ draws_[nvalues * j + i] = draws_[i];
+ }
+ }
+ }
+
+ void ExecuteSpecialValues(int nvalues, int repeats) {
+ InitDataWithSpecialValues(nvalues, repeats);
+ CheckRoundtrip();
+ }
+
+ void ExecuteDecimalPattern(int nvalues, int repeats) {
+ InitDataDecimalPattern(nvalues, repeats);
+ CheckRoundtrip();
+ }
+
+ protected:
+ USING_BASE_MEMBERS();
+};
+
+using AlpEncodedTypes = ::testing::Types<FloatType, DoubleType>;
+TYPED_TEST_SUITE(TestAlpEncoding, AlpEncodedTypes);
+
+TYPED_TEST(TestAlpEncoding, BasicRoundTrip) {
+ // Test various sizes including edge cases
+ for (int values = 1; values < 32; ++values) {
+ ASSERT_NO_FATAL_FAILURE(this->Execute(values, 1));
+ }
+
+ // Test exactly vector size (1024)
+ ASSERT_NO_FATAL_FAILURE(this->Execute(1024, 1));
+
+ // Test just under and over vector size
+ ASSERT_NO_FATAL_FAILURE(this->Execute(1023, 1));
+ ASSERT_NO_FATAL_FAILURE(this->Execute(1025, 1));
+
+ // Test multiple vectors
+ ASSERT_NO_FATAL_FAILURE(this->Execute(2048, 1));
+ ASSERT_NO_FATAL_FAILURE(this->Execute(3000, 1));
+}
+
+TYPED_TEST(TestAlpEncoding, RoundTripWithRepeats) {
+ // Test with repeated patterns
+ ASSERT_NO_FATAL_FAILURE(this->Execute(100, 10));
+ ASSERT_NO_FATAL_FAILURE(this->Execute(1024, 3));
+}
+
+TYPED_TEST(TestAlpEncoding, SpecialValues) {
+ // Test NaN, Inf, -Inf, -0.0 (these become exceptions in ALP)
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteSpecialValues(100, 1));
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteSpecialValues(1024, 1));
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteSpecialValues(2000, 1));
+}
+
+TYPED_TEST(TestAlpEncoding, DecimalPatterns) {
+ // Test decimal-like values that ALP compresses efficiently
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteDecimalPattern(100, 1));
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteDecimalPattern(1024, 1));
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteDecimalPattern(5000, 1));
+}
+
+TYPED_TEST(TestAlpEncoding, SpacedRoundTrip) {
+ // Test with null values at various probabilities
+ for (double null_prob : {0.0, 0.1, 0.5, 0.9}) {
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteSpaced(100, 1, 0, null_prob));
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteSpaced(1024, 1, 0, null_prob));
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteSpaced(2000, 1, 0, null_prob));
+ }
+
+ // Test with offset
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteSpaced(1024, 1, 7, 0.3));
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteSpaced(1024, 1, 64, 0.5));
+}
+
+TYPED_TEST(TestAlpEncoding, LargeDataset) {
+ // Test with large dataset (multiple pages worth)
+ ASSERT_NO_FATAL_FAILURE(this->Execute(100000, 1));
+}
+
+TYPED_TEST(TestAlpEncoding, RandomData) {
+ using c_type = typename TypeParam::c_type;
+ ::arrow::random::RandomArrayGenerator rag(42);
+
+ // Generate random float/double array
+ std::shared_ptr<::arrow::Array> arr;
+ if constexpr (std::is_same_v<c_type, float>) {
+ arr = rag.Float32(10000, -1000.0f, 1000.0f);
+ } else {
+ arr = rag.Float64(10000, -1000.0, 1000.0);
+ }
+
+ auto encoder = MakeTypedEncoder<TypeParam>(Encoding::ALP, false,
this->descr_.get());
+ ASSERT_NO_THROW(encoder->Put(*arr));
+ auto buffer = encoder->FlushValues();
+
+ auto decoder = MakeTypedDecoder<TypeParam>(Encoding::ALP,
this->descr_.get());
+ decoder->SetData(static_cast<int>(arr->length()), buffer->data(),
+ static_cast<int>(buffer->size()));
+
+ std::vector<c_type> output(arr->length());
+ int decoded = decoder->Decode(output.data(),
static_cast<int>(arr->length()));
+ ASSERT_EQ(decoded, arr->length());
+
+ // Verify round-trip
+ auto typed_arr = std::static_pointer_cast<
+ typename std::conditional<std::is_same_v<c_type, float>,
+ ::arrow::FloatArray,
::arrow::DoubleArray>::type>(arr);
+ ASSERT_EQ(0, std::memcmp(output.data(), typed_arr->raw_values(),
+ arr->length() * sizeof(c_type)));
+}
+
+TEST(AlpEncodingAdHoc, InvalidDataTypes) {
+ // ALP only supports float and double
+ ASSERT_THROW(MakeTypedEncoder<Int32Type>(Encoding::ALP), ParquetException);
+ ASSERT_THROW(MakeTypedEncoder<Int64Type>(Encoding::ALP), ParquetException);
+ ASSERT_THROW(MakeTypedEncoder<BooleanType>(Encoding::ALP), ParquetException);
+ ASSERT_THROW(MakeTypedEncoder<ByteArrayType>(Encoding::ALP),
ParquetException);
+
+ ASSERT_THROW(MakeTypedDecoder<Int32Type>(Encoding::ALP), ParquetException);
+ ASSERT_THROW(MakeTypedDecoder<Int64Type>(Encoding::ALP), ParquetException);
+ ASSERT_THROW(MakeTypedDecoder<BooleanType>(Encoding::ALP), ParquetException);
+ ASSERT_THROW(MakeTypedDecoder<ByteArrayType>(Encoding::ALP),
ParquetException);
+}
+
+TEST(AlpEncodingAdHoc, ConstantValues) {
Review Comment:
Folded — `NonDefaultVectorSizeRoundTrip` is a `TYPED_TEST` over float and
double now, so the duplicated per-type blocks are gone. The one plain `TEST`
left is `InvalidDataTypes`, which asserts that the non-float types are
rejected, so it can't be parameterized over the float types.
##########
cpp/src/parquet/encoding_test.cc:
##########
@@ -2660,4 +2595,407 @@ TEST(DeltaByteArrayEncodingAdHoc, ArrowDirectPut) {
}
}
+// ----------------------------------------------------------------------
+// ALP encoding tests for float/double
+
+template <typename Type>
+class TestAlpEncoding : public TestEncodingBase<Type> {
+ public:
+ using c_type = typename Type::c_type;
+ static constexpr int TYPE = Type::type_num;
+ static constexpr size_t kNumRoundTrips = 3;
+
+ void CheckRoundtrip() override {
+ auto encoder =
+ MakeTypedEncoder<Type>(Encoding::ALP, /*use_dictionary=*/false,
descr_.get());
+ auto decoder = MakeTypedDecoder<Type>(Encoding::ALP, descr_.get());
+
+ for (size_t i = 0; i < kNumRoundTrips; ++i) {
+ encoder->Put(draws_, num_values_);
+ encode_buffer_ = encoder->FlushValues();
+
+ decoder->SetData(num_values_, encode_buffer_->data(),
+ static_cast<int>(encode_buffer_->size()));
+ int values_decoded = decoder->Decode(decode_buf_, num_values_);
+ ASSERT_EQ(num_values_, values_decoded);
+
+ // Use memcmp for bit-exact comparison (important for -0.0, NaN bit
patterns)
+ ASSERT_EQ(0, std::memcmp(draws_, decode_buf_, num_values_ *
sizeof(c_type)));
+ }
+ }
+
+ void CheckRoundtripSpaced(const uint8_t* valid_bits,
+ int64_t valid_bits_offset) override {
+ auto encoder =
+ MakeTypedEncoder<Type>(Encoding::ALP, /*use_dictionary=*/false,
descr_.get());
+ auto decoder = MakeTypedDecoder<Type>(Encoding::ALP, descr_.get());
+
+ int null_count = 0;
+ for (auto i = 0; i < num_values_; i++) {
+ if (!bit_util::GetBit(valid_bits, valid_bits_offset + i)) {
+ null_count++;
+ }
+ }
+
+ for (size_t i = 0; i < kNumRoundTrips; ++i) {
+ encoder->PutSpaced(draws_, num_values_, valid_bits, valid_bits_offset);
+ encode_buffer_ = encoder->FlushValues();
+
+ decoder->SetData(num_values_ - null_count, encode_buffer_->data(),
+ static_cast<int>(encode_buffer_->size()));
+ auto values_decoded = decoder->DecodeSpaced(decode_buf_, num_values_,
null_count,
+ valid_bits,
valid_bits_offset);
+ ASSERT_EQ(num_values_, values_decoded);
+
+ // Verify only valid values
+ for (int j = 0; j < num_values_; ++j) {
+ if (bit_util::GetBit(valid_bits, valid_bits_offset + j)) {
+ ASSERT_EQ(0, std::memcmp(&draws_[j], &decode_buf_[j],
sizeof(c_type))) << j;
+ }
+ }
+ }
+ }
+
+ void InitDataWithSpecialValues(int nvalues, int repeats) {
+ num_values_ = nvalues * repeats;
+ this->input_bytes_.resize(num_values_ * sizeof(c_type));
+ this->output_bytes_.resize(num_values_ * sizeof(c_type));
+ draws_ = reinterpret_cast<c_type*>(this->input_bytes_.data());
+ decode_buf_ = reinterpret_cast<c_type*>(this->output_bytes_.data());
+
+ // Fill with mix of normal and special values
+ for (int i = 0; i < nvalues; ++i) {
+ if (i % 20 == 0) {
+ draws_[i] = std::numeric_limits<c_type>::quiet_NaN();
+ } else if (i % 20 == 5) {
+ draws_[i] = std::numeric_limits<c_type>::infinity();
+ } else if (i % 20 == 10) {
+ draws_[i] = -std::numeric_limits<c_type>::infinity();
+ } else if (i % 20 == 15) {
+ draws_[i] = static_cast<c_type>(-0.0);
+ } else {
+ draws_[i] = static_cast<c_type>(i) * static_cast<c_type>(0.123);
+ }
+ }
+
+ // Repeat pattern
+ for (int j = 1; j < repeats; ++j) {
+ for (int i = 0; i < nvalues; ++i) {
+ draws_[nvalues * j + i] = draws_[i];
+ }
+ }
+ }
+
+ void InitDataDecimalPattern(int nvalues, int repeats) {
+ num_values_ = nvalues * repeats;
+ this->input_bytes_.resize(num_values_ * sizeof(c_type));
+ this->output_bytes_.resize(num_values_ * sizeof(c_type));
+ draws_ = reinterpret_cast<c_type*>(this->input_bytes_.data());
+ decode_buf_ = reinterpret_cast<c_type*>(this->output_bytes_.data());
+
+ // Decimal-like values that ALP compresses well
+ for (int i = 0; i < nvalues; ++i) {
+ draws_[i] = static_cast<c_type>(100.0 + i * 0.01);
+ }
+
+ for (int j = 1; j < repeats; ++j) {
+ for (int i = 0; i < nvalues; ++i) {
+ draws_[nvalues * j + i] = draws_[i];
+ }
+ }
+ }
+
+ void ExecuteSpecialValues(int nvalues, int repeats) {
+ InitDataWithSpecialValues(nvalues, repeats);
+ CheckRoundtrip();
+ }
+
+ void ExecuteDecimalPattern(int nvalues, int repeats) {
+ InitDataDecimalPattern(nvalues, repeats);
+ CheckRoundtrip();
+ }
+
+ protected:
+ USING_BASE_MEMBERS();
+};
+
+using AlpEncodedTypes = ::testing::Types<FloatType, DoubleType>;
+TYPED_TEST_SUITE(TestAlpEncoding, AlpEncodedTypes);
+
+TYPED_TEST(TestAlpEncoding, BasicRoundTrip) {
+ // Test various sizes including edge cases
+ for (int values = 1; values < 32; ++values) {
+ ASSERT_NO_FATAL_FAILURE(this->Execute(values, 1));
+ }
+
+ // Test exactly vector size (1024)
+ ASSERT_NO_FATAL_FAILURE(this->Execute(1024, 1));
+
+ // Test just under and over vector size
+ ASSERT_NO_FATAL_FAILURE(this->Execute(1023, 1));
+ ASSERT_NO_FATAL_FAILURE(this->Execute(1025, 1));
+
+ // Test multiple vectors
+ ASSERT_NO_FATAL_FAILURE(this->Execute(2048, 1));
+ ASSERT_NO_FATAL_FAILURE(this->Execute(3000, 1));
+}
+
+TYPED_TEST(TestAlpEncoding, RoundTripWithRepeats) {
+ // Test with repeated patterns
+ ASSERT_NO_FATAL_FAILURE(this->Execute(100, 10));
+ ASSERT_NO_FATAL_FAILURE(this->Execute(1024, 3));
+}
+
+TYPED_TEST(TestAlpEncoding, SpecialValues) {
+ // Test NaN, Inf, -Inf, -0.0 (these become exceptions in ALP)
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteSpecialValues(100, 1));
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteSpecialValues(1024, 1));
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteSpecialValues(2000, 1));
+}
+
+TYPED_TEST(TestAlpEncoding, DecimalPatterns) {
+ // Test decimal-like values that ALP compresses efficiently
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteDecimalPattern(100, 1));
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteDecimalPattern(1024, 1));
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteDecimalPattern(5000, 1));
+}
+
+TYPED_TEST(TestAlpEncoding, SpacedRoundTrip) {
+ // Test with null values at various probabilities
+ for (double null_prob : {0.0, 0.1, 0.5, 0.9}) {
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteSpaced(100, 1, 0, null_prob));
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteSpaced(1024, 1, 0, null_prob));
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteSpaced(2000, 1, 0, null_prob));
+ }
+
+ // Test with offset
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteSpaced(1024, 1, 7, 0.3));
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteSpaced(1024, 1, 64, 0.5));
+}
+
+TYPED_TEST(TestAlpEncoding, LargeDataset) {
+ // Test with large dataset (multiple pages worth)
+ ASSERT_NO_FATAL_FAILURE(this->Execute(100000, 1));
+}
+
+TYPED_TEST(TestAlpEncoding, RandomData) {
+ using c_type = typename TypeParam::c_type;
+ ::arrow::random::RandomArrayGenerator rag(42);
+
+ // Generate random float/double array
+ std::shared_ptr<::arrow::Array> arr;
+ if constexpr (std::is_same_v<c_type, float>) {
+ arr = rag.Float32(10000, -1000.0f, 1000.0f);
+ } else {
+ arr = rag.Float64(10000, -1000.0, 1000.0);
+ }
+
+ auto encoder = MakeTypedEncoder<TypeParam>(Encoding::ALP, false,
this->descr_.get());
+ ASSERT_NO_THROW(encoder->Put(*arr));
+ auto buffer = encoder->FlushValues();
+
+ auto decoder = MakeTypedDecoder<TypeParam>(Encoding::ALP,
this->descr_.get());
+ decoder->SetData(static_cast<int>(arr->length()), buffer->data(),
+ static_cast<int>(buffer->size()));
+
+ std::vector<c_type> output(arr->length());
+ int decoded = decoder->Decode(output.data(),
static_cast<int>(arr->length()));
+ ASSERT_EQ(decoded, arr->length());
+
+ // Verify round-trip
+ auto typed_arr = std::static_pointer_cast<
+ typename std::conditional<std::is_same_v<c_type, float>,
+ ::arrow::FloatArray,
::arrow::DoubleArray>::type>(arr);
+ ASSERT_EQ(0, std::memcmp(output.data(), typed_arr->raw_values(),
+ arr->length() * sizeof(c_type)));
+}
+
+TEST(AlpEncodingAdHoc, InvalidDataTypes) {
+ // ALP only supports float and double
+ ASSERT_THROW(MakeTypedEncoder<Int32Type>(Encoding::ALP), ParquetException);
+ ASSERT_THROW(MakeTypedEncoder<Int64Type>(Encoding::ALP), ParquetException);
+ ASSERT_THROW(MakeTypedEncoder<BooleanType>(Encoding::ALP), ParquetException);
+ ASSERT_THROW(MakeTypedEncoder<ByteArrayType>(Encoding::ALP),
ParquetException);
+
+ ASSERT_THROW(MakeTypedDecoder<Int32Type>(Encoding::ALP), ParquetException);
+ ASSERT_THROW(MakeTypedDecoder<Int64Type>(Encoding::ALP), ParquetException);
+ ASSERT_THROW(MakeTypedDecoder<BooleanType>(Encoding::ALP), ParquetException);
+ ASSERT_THROW(MakeTypedDecoder<ByteArrayType>(Encoding::ALP),
ParquetException);
+}
+
+TEST(AlpEncodingAdHoc, ConstantValues) {
+ // Test all same values (should compress to bit_width=0)
+ auto descr = ExampleDescr<DoubleType>();
+ std::vector<double> data(1024, 123.456);
+
+ auto encoder = MakeTypedEncoder<DoubleType>(Encoding::ALP, false,
descr.get());
+ encoder->Put(data.data(), static_cast<int>(data.size()));
+ auto buffer = encoder->FlushValues();
+
+ auto decoder = MakeTypedDecoder<DoubleType>(Encoding::ALP, descr.get());
+ decoder->SetData(static_cast<int>(data.size()), buffer->data(),
+ static_cast<int>(buffer->size()));
+
+ std::vector<double> output(data.size());
+ int decoded = decoder->Decode(output.data(), static_cast<int>(data.size()));
+ ASSERT_EQ(decoded, static_cast<int>(data.size()));
+
+ for (size_t i = 0; i < data.size(); ++i) {
+ ASSERT_EQ(data[i], output[i]) << i;
+ }
+}
+
+TEST(AlpEncodingAdHoc, AllExceptions) {
Review Comment:
Same answer — folded into the type-parameterized suite.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
