This is an automated email from the ASF dual-hosted git repository.
HuaHuaY pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 84472e46c9 GH-52161: [C++][Parquet] Add support for string/binary view
to content defined chunking writer (#52162)
84472e46c9 is described below
commit 84472e46c93dc87af738704f91fea6e6b6e2cf02
Author: Robert Kruszewski <[email protected]>
AuthorDate: Mon Oct 12 05:35:45 2026 +0100
GH-52161: [C++][Parquet] Add support for string/binary view to content
defined chunking writer (#52162)
### Rationale for this change
View types are supported in parquet writer but they error when using
content defined chunking
### What changes are included in this PR?
Teach the internal chunker how to calculate the size for view types
### Are these changes tested?
Added tests
### Are there any user-facing changes?
No
### Was AI used for this PR?
In accordance to the [AI generation
guidelines](https://arrow.apache.org/docs/dev/developers/overview.html#ai-generated-code),
please disclose below whether and how AI was used in this PR.
**PR code and description written by:**
- [x] Human
- [ ] AI
**Reviewed before submission by:**
- [x] Human
- [x] AI
- [ ] Not reviewed
* GitHub Issue: #52161
Authored-by: Robert Kruszewski <[email protected]>
Signed-off-by: Zehua Zou <[email protected]>
---
cpp/src/parquet/chunker_internal.cc | 14 ++++++++++++++
cpp/src/parquet/chunker_internal_test.cc | 15 +++++++++++++--
2 files changed, 27 insertions(+), 2 deletions(-)
diff --git a/cpp/src/parquet/chunker_internal.cc
b/cpp/src/parquet/chunker_internal.cc
index 794075b733..c45bf0c97e 100644
--- a/cpp/src/parquet/chunker_internal.cc
+++ b/cpp/src/parquet/chunker_internal.cc
@@ -21,6 +21,7 @@
#include <cstdint>
#include <iterator>
#include <string>
+#include <string_view>
#include <vector>
#include "arrow/array.h"
@@ -349,6 +350,17 @@ class ContentDefinedChunker::Impl {
});
}
+ std::vector<Chunk> CalculateBinaryView(const int16_t* def_levels,
+ const int16_t* rep_levels, int64_t
num_levels,
+ const ::arrow::Array& values) {
+ const auto& array = checked_cast<const ::arrow::BinaryViewArray&>(values);
+ return Calculate(def_levels, rep_levels, num_levels, [&](int64_t i) {
+ const std::string_view value = array.GetView(i);
+ Roll(reinterpret_cast<const uint8_t*>(value.data()),
+ static_cast<int64_t>(value.size()));
+ });
+ }
+
std::vector<Chunk> GetChunks(const int16_t* def_levels, const int16_t*
rep_levels,
int64_t num_levels, const ::arrow::Array&
values) {
auto handle_type = [&](auto&& type) -> std::vector<Chunk> {
@@ -380,6 +392,8 @@ class ContentDefinedChunker::Impl {
} else if constexpr (::arrow::is_large_binary_like(ArrowType::type_id)) {
return CalculateBinaryLike<::arrow::LargeBinaryArray>(def_levels,
rep_levels,
num_levels,
values);
+ } else if constexpr (::arrow::is_binary_view_like(ArrowType::type_id)) {
+ return CalculateBinaryView(def_levels, rep_levels, num_levels, values);
} else if constexpr (::arrow::is_dictionary(ArrowType::type_id)) {
return GetChunks(def_levels, rep_levels, num_levels,
*static_cast<const
::arrow::DictionaryArray&>(values).indices());
diff --git a/cpp/src/parquet/chunker_internal_test.cc
b/cpp/src/parquet/chunker_internal_test.cc
index 2469d54afb..8de4136cfa 100644
--- a/cpp/src/parquet/chunker_internal_test.cc
+++ b/cpp/src/parquet/chunker_internal_test.cc
@@ -30,6 +30,7 @@
#include "arrow/testing/generator.h"
#include "arrow/type_fwd.h"
#include "arrow/util/float16.h"
+#include "arrow/visit_data_inline.h"
#include "parquet/arrow/reader.h"
#include "parquet/arrow/reader_internal.h"
#include "parquet/arrow/test_util.h"
@@ -170,6 +171,10 @@ Result<std::shared_ptr<Array>> GenerateArray(const
std::shared_ptr<Field>& field
std::string("bin_") + std::to_string(val))
GENERATE_CASE(LARGE_BINARY, ::arrow::LargeBinaryBuilder,
std::string("bin_") + std::to_string(val))
+ GENERATE_CASE(STRING_VIEW, ::arrow::StringViewBuilder,
+ std::string("str_") + std::to_string(val))
+ GENERATE_CASE(BINARY_VIEW, ::arrow::BinaryViewBuilder,
+ std::string("bin_") + std::to_string(val))
case ::arrow::Type::FIXED_SIZE_BINARY: {
auto size =
checked_cast<const
::arrow::FixedSizeBinaryType*>(type.get())->byte_width();
@@ -742,6 +747,10 @@ Result<int64_t> CalculateCdcSize(const
std::shared_ptr<Array>& array, bool nulla
} else if (::arrow::is_large_binary_like(type_id)) {
auto binary_array = checked_cast<const
::arrow::LargeBinaryArray*>(array.get());
result += binary_array->total_values_length();
+ } else if (::arrow::is_binary_view_like(type_id)) {
+ ::arrow::VisitArraySpanInline<::arrow::BinaryViewType>(
+ *array->data(),
+ [&](std::string_view v) { result += static_cast<int64_t>(v.size()); },
[] {});
} else {
return Status::NotImplemented("CDC size calculation for type ",
array->type()->ToString(), " is not
implemented");
@@ -784,7 +793,8 @@ void AssertContentDefinedChunkSizes(const
std::shared_ptr<::arrow::ChunkedArray>
ASSERT_EQ(column_info.has_dictionary_page, expect_dictionary_page);
}
- if (::arrow::is_fixed_width(type_id) ||
::arrow::is_base_binary_like(type_id)) {
+ if (::arrow::is_fixed_width(type_id) ||
::arrow::is_base_binary_like(type_id) ||
+ ::arrow::is_binary_view_like(type_id)) {
int64_t offset = 0;
auto page_lengths = column_info.page_lengths;
@@ -1451,7 +1461,7 @@ INSTANTIATE_TEST_SUITE_P(
testing::Values(
CaseConfig{::arrow::boolean(), false}, CaseConfig{::arrow::int64(),
true},
// Binary-like
- CaseConfig{::arrow::utf8(), false},
+ CaseConfig{::arrow::utf8(), false}, CaseConfig{::arrow::utf8_view(),
true},
CaseConfig{::arrow::fixed_size_binary(16), true},
// Nested types
CaseConfig{::arrow::list(::arrow::int32()), false},
@@ -1475,6 +1485,7 @@ INSTANTIATE_TEST_SUITE_P(
// Binary-like
CaseConfig{::arrow::utf8(), false}, CaseConfig{::arrow::binary(),
true},
CaseConfig{::arrow::fixed_size_binary(16), true},
+ CaseConfig{::arrow::utf8_view(), false},
CaseConfig{::arrow::binary_view(), true},
// Temporal
CaseConfig{::arrow::date32(), false},
CaseConfig{::arrow::time32(::arrow::TimeUnit::MILLI), true},
