This is an automated email from the ASF dual-hosted git repository.
pitrou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new cffe22d6fe GH-38849: [C++][Parquet] Add support for list view and
large list view (#50160)
cffe22d6fe is described below
commit cffe22d6fe7f73836d2d9177b4b4fb2d1fcf4d1f
Author: Zehua Zou <[email protected]>
AuthorDate: Thu Jun 25 20:25:31 2026 +0800
GH-38849: [C++][Parquet] Add support for list view and large list view
(#50160)
### Rationale for this change
Allow us to write an arrow list view array into a parquet file and read
list view type from parquet.
### What changes are included in this PR?
We will sort the list views based on the offsets before inserting the array
in Parquet writer, and generate a size array in Parquet reader.
### Are these changes tested?
Yes.
### Are there any user-facing changes?
No.
* GitHub Issue: #38849
Authored-by: Zehua Zou <[email protected]>
Signed-off-by: Antoine Pitrou <[email protected]>
---
cpp/src/parquet/arrow/arrow_reader_writer_test.cc | 71 +++++++++++
cpp/src/parquet/arrow/arrow_schema_test.cc | 53 ++++----
cpp/src/parquet/arrow/path_internal.cc | 140 ++++++++++++++--------
cpp/src/parquet/arrow/path_internal_test.cc | 25 ++++
cpp/src/parquet/arrow/reader.cc | 53 +++++++-
cpp/src/parquet/arrow/schema.cc | 52 ++++----
cpp/src/parquet/arrow/writer.cc | 24 +++-
cpp/src/parquet/properties.h | 3 +-
docs/source/cpp/parquet.rst | 3 +-
9 files changed, 312 insertions(+), 112 deletions(-)
diff --git a/cpp/src/parquet/arrow/arrow_reader_writer_test.cc
b/cpp/src/parquet/arrow/arrow_reader_writer_test.cc
index d29458bf22..8735aea731 100644
--- a/cpp/src/parquet/arrow/arrow_reader_writer_test.cc
+++ b/cpp/src/parquet/arrow/arrow_reader_writer_test.cc
@@ -28,9 +28,11 @@
#include <functional>
#include <set>
#include <sstream>
+#include <type_traits>
#include <utility>
#include <vector>
+#include "arrow/array/array_nested.h"
#include "arrow/array/builder_binary.h"
#include "arrow/array/builder_decimal.h"
#include "arrow/array/builder_dict.h"
@@ -3324,6 +3326,75 @@ TEST(ArrowReadWrite, LargeList) {
}
}
+template <typename ViewType>
+ requires ::arrow::is_list_view_type<ViewType>::value
+void CheckListViewRoundTrip(
+ const std::shared_ptr<ViewType>& view_type,
+ const std::shared_ptr<::arrow::DataType>& fallback_type,
+ const ArrowReaderProperties& reader_props =
default_arrow_reader_properties()) {
+ using ViewArrayType = typename ::arrow::TypeTraits<ViewType>::ArrayType;
+ using OffsetArrowType = typename ::arrow::TypeTraits<ViewType>::OffsetType;
+ using FallbackArrayType =
+ std::conditional_t<std::is_same_v<ViewType, ::arrow::ListViewType>,
+ ::arrow::ListArray, ::arrow::LargeListArray>;
+
+ auto values = ArrayFromJSON(::arrow::int32(), "[1, 2, 3, 4, 5]");
+ auto offsets =
ArrayFromJSON(::arrow::TypeTraits<OffsetArrowType>::type_singleton(),
+ "[3, 0, 5, 1]");
+ auto sizes =
ArrayFromJSON(::arrow::TypeTraits<OffsetArrowType>::type_singleton(),
+ "[2, 1, 0, 2]");
+ ASSERT_OK_AND_ASSIGN(auto array,
+ ViewArrayType::FromArrays(view_type, *offsets, *sizes,
*values,
+ default_memory_pool()));
+
+ auto table = Table::Make(
+ ::arrow::schema({::arrow::field("root", array->type(), false)}),
{array});
+
+ auto props_store_schema =
ArrowWriterProperties::Builder().store_schema()->build();
+ CheckSimpleRoundtrip(table, 2, props_store_schema);
+
+ ASSERT_OK_AND_ASSIGN(auto expected_array,
+ FallbackArrayType::FromListView(*array,
default_memory_pool()));
+
+ auto expected = Table::Make(
+ ::arrow::schema({::arrow::field("root", fallback_type, false)}),
{expected_array});
+ CheckConfiguredRoundtrip(table, expected,
::parquet::default_writer_properties(),
+ default_arrow_writer_properties(), reader_props);
+}
+
+TEST(ArrowReadWrite, ListView) {
+
CheckListViewRoundTrip(std::make_shared<::arrow::ListViewType>(::arrow::int32()),
+ ::arrow::list(::arrow::int32()));
+}
+
+TEST(ArrowReadWrite, LargeListView) {
+ ArrowReaderProperties reader_props;
+ reader_props.set_list_type(::arrow::Type::LARGE_LIST);
+
CheckListViewRoundTrip(std::make_shared<::arrow::LargeListViewType>(::arrow::int32()),
+ ::arrow::large_list(::arrow::int32()), reader_props);
+}
+
+TEST(ArrowReadWrite, EmptyListView) {
+ auto type = ::arrow::list_view(::arrow::int32());
+ auto array = ArrayFromJSON(type, "[]");
+ auto table = Table::Make(::arrow::schema({::arrow::field("root", type)}),
{array});
+
+ auto props_store_schema =
ArrowWriterProperties::Builder().store_schema()->build();
+ std::shared_ptr<Table> result;
+ ASSERT_NO_FATAL_FAILURE(
+ DoRoundtrip(table, 1, &result, default_writer_properties(),
props_store_schema));
+ ASSERT_OK(result->ValidateFull());
+
+ ASSERT_EQ(1, result->column(0)->num_chunks());
+ const auto& list_view =
+ checked_cast<const
::arrow::ListViewArray&>(*result->column(0)->chunk(0));
+ ASSERT_EQ(0, list_view.length());
+ ASSERT_NE(nullptr, list_view.value_offsets());
+ ASSERT_EQ(0, list_view.value_offsets()->size());
+ ASSERT_NE(nullptr, list_view.value_sizes());
+ ASSERT_EQ(0, list_view.value_sizes()->size());
+}
+
TEST(ArrowReadWrite, FixedSizeList) {
using ::arrow::field;
using ::arrow::fixed_size_list;
diff --git a/cpp/src/parquet/arrow/arrow_schema_test.cc
b/cpp/src/parquet/arrow/arrow_schema_test.cc
index 7d9ecb5e64..7a7b5a3369 100644
--- a/cpp/src/parquet/arrow/arrow_schema_test.cc
+++ b/cpp/src/parquet/arrow/arrow_schema_test.cc
@@ -1728,40 +1728,35 @@ TEST_F(TestConvertArrowSchema, ParquetOtherLists) {
std::vector<NodePtr> parquet_fields;
std::vector<std::shared_ptr<Field>> arrow_fields;
- // parquet_arrow will always generate 3-level LIST encodings
-
- // // LargeList<String> (list-like non-null, elements nullable)
- // required group my_list (LIST) {
- // repeated group list {
- // optional binary element (UTF8);
- // }
- // }
- {
- auto element = PrimitiveNode::Make("element", Repetition::OPTIONAL,
- ParquetType::BYTE_ARRAY,
ConvertedType::UTF8);
- auto list = GroupNode::Make("list", Repetition::REPEATED, {element});
- parquet_fields.push_back(
- GroupNode::Make("my_list", Repetition::REQUIRED, {list},
ConvertedType::LIST));
- auto arrow_element = ::arrow::field("string", UTF8, true);
- auto arrow_list = ::arrow::large_list(arrow_element);
- arrow_fields.push_back(::arrow::field("my_list", arrow_list, false));
- }
- // // FixedSizeList[10]<String> (list-like non-null, elements nullable)
+ // parquet_arrow will always generate this 3-level LIST encoding for
list-like
+ // non-null Arrow arrays with nullable string elements:
+ //
// required group my_list (LIST) {
// repeated group list {
// optional binary element (UTF8);
// }
// }
- {
- auto element = PrimitiveNode::Make("element", Repetition::OPTIONAL,
- ParquetType::BYTE_ARRAY,
ConvertedType::UTF8);
- auto list = GroupNode::Make("list", Repetition::REPEATED, {element});
- parquet_fields.push_back(
- GroupNode::Make("my_list", Repetition::REQUIRED, {list},
ConvertedType::LIST));
- auto arrow_element = ::arrow::field("string", UTF8, true);
- auto arrow_list = ::arrow::fixed_size_list(arrow_element, 10);
- arrow_fields.push_back(::arrow::field("my_list", arrow_list, false));
- }
+ auto element = PrimitiveNode::Make("element", Repetition::OPTIONAL,
+ ParquetType::BYTE_ARRAY,
ConvertedType::UTF8);
+ auto list = GroupNode::Make("list", Repetition::REPEATED, {element});
+ auto parquet_field =
+ GroupNode::Make("my_list", Repetition::REQUIRED, {list},
ConvertedType::LIST);
+
+ auto AddListLikeField = [&](std::shared_ptr<::arrow::DataType> arrow_list) {
+ parquet_fields.push_back(parquet_field);
+ arrow_fields.push_back(::arrow::field("my_list", std::move(arrow_list),
false));
+ };
+
+ auto arrow_element = ::arrow::field("string", UTF8, true);
+
+ // LargeList<String> (list-like non-null, elements nullable)
+ AddListLikeField(::arrow::large_list(arrow_element));
+ // ListView<String> (list-like non-null, elements nullable)
+ AddListLikeField(::arrow::list_view(arrow_element));
+ // LargeListView<String> (list-like non-null, elements nullable)
+ AddListLikeField(::arrow::large_list_view(arrow_element));
+ // FixedSizeList[10]<String> (list-like non-null, elements nullable)
+ AddListLikeField(::arrow::fixed_size_list(arrow_element, 10));
ASSERT_OK(ConvertSchema(arrow_fields));
diff --git a/cpp/src/parquet/arrow/path_internal.cc
b/cpp/src/parquet/arrow/path_internal.cc
index 002859a5e7..c626dcbf48 100644
--- a/cpp/src/parquet/arrow/path_internal.cc
+++ b/cpp/src/parquet/arrow/path_internal.cc
@@ -387,34 +387,41 @@ class ListPathNode {
PathWriteContext* context) {
// First fill int the remainder of the list.
RETURN_IF_ERROR(FillRepLevels(child_range->Size(), rep_level_, context));
+
// Once we've reached this point the following preconditions should hold:
// 1. There are no more repeated path nodes to deal with.
- // 2. All elements in |range| represent contiguous elements in the
- // child array (Null values would have shortened the range to ensure
- // all remaining list elements are present (though they may be empty
lists)).
+ // 2. Null values would have shortened the range to ensure all remaining
+ // list elements are present (though they may be empty lists).
// 3. No element of range spans a parent list (intermediate
// list nodes only handle one list entry at a time).
//
- // Given these preconditions it should be safe to fill runs on non-empty
+ // Given these preconditions it is safe to fill runs on contiguous
non-empty
// lists here and expand the range in the child node accordingly.
-
while (!range->Empty()) {
- ElementRange size_check = selector_.GetRange(range->start);
- if (size_check.Empty()) {
+ ElementRange next_child_range = selector_.GetRange(range->start);
+ if (next_child_range.Empty()) {
// The empty range will need to be handled after we pass down the
accumulated
// range because it affects def_level placement and we need to get the
children
// def_levels entered first.
break;
}
+ // FillForLast extends child_range by updating only its end.
Non-contiguous
+ // selectors must split at gaps.
+ if constexpr (RangeSelector::kContiguous) {
+ DCHECK_EQ(next_child_range.start, child_range->end)
+ << next_child_range.start << " != " << child_range->end;
+ } else {
+ if (next_child_range.start != child_range->end) {
+ break;
+ }
+ }
// This is the start of a new list. We can be sure it only applies
// to the previous list (and doesn't jump to the start of any list
// further up in nesting due to the constraints mentioned at the start
// of the function).
RETURN_IF_ERROR(context->AppendRepLevel(prev_rep_level_));
- RETURN_IF_ERROR(context->AppendRepLevels(size_check.Size() - 1,
rep_level_));
- DCHECK_EQ(size_check.start, child_range->end)
- << size_check.start << " != " << child_range->end;
- child_range->end = size_check.end;
+ RETURN_IF_ERROR(context->AppendRepLevels(next_child_range.Size() - 1,
rep_level_));
+ child_range->end = next_child_range.end;
++range->start;
}
@@ -434,19 +441,37 @@ class ListPathNode {
template <typename OffsetType>
struct VarRangeSelector {
+ static constexpr bool kContiguous = true;
+
ElementRange GetRange(int64_t index) const {
- return ElementRange{offsets[index], offsets[index + 1]};
+ return ElementRange{.start = offsets[index], .end = offsets[index + 1]};
}
// Either int32_t* or int64_t*.
const OffsetType* offsets;
};
+template <typename OffsetType>
+struct ListViewRangeSelector {
+ static constexpr bool kContiguous = false;
+
+ ElementRange GetRange(int64_t index) const {
+ const int64_t start = offsets[index];
+ return ElementRange{.start = start, .end = start + sizes[index]};
+ }
+
+ const OffsetType* offsets;
+ const OffsetType* sizes;
+};
+
struct FixedSizedRangeSelector {
+ static constexpr bool kContiguous = true;
+
ElementRange GetRange(int64_t index) const {
int64_t start = index * list_size;
- return ElementRange{start, start + list_size};
+ return ElementRange{.start = start, .end = start + list_size};
}
+
int list_size;
};
@@ -510,6 +535,8 @@ class NullableNode {
using ListNode = ListPathNode<VarRangeSelector<int32_t>>;
using LargeListNode = ListPathNode<VarRangeSelector<int64_t>>;
+using ListViewNode = ListPathNode<ListViewRangeSelector<int32_t>>;
+using LargeListViewNode = ListPathNode<ListViewRangeSelector<int64_t>>;
using FixedSizeListNode = ListPathNode<FixedSizedRangeSelector>;
// Contains static information derived from traversing the schema.
@@ -517,9 +544,9 @@ struct PathInfo {
// The vectors are expected to the same length info.
// Note index order matters here.
- using Node =
- std::variant<NullableTerminalNode, ListNode, LargeListNode,
FixedSizeListNode,
- NullableNode, AllPresentTerminalNode, AllNullsTerminalNode>;
+ using Node = std::variant<NullableTerminalNode, ListNode, LargeListNode,
ListViewNode,
+ LargeListViewNode, FixedSizeListNode, NullableNode,
+ AllPresentTerminalNode, AllNullsTerminalNode>;
std::vector<Node> path;
std::shared_ptr<Array> primitive_array;
@@ -529,6 +556,28 @@ struct PathInfo {
bool leaf_is_nullable = false;
};
+struct WritePathVisitor {
+ IterationResult operator()(NullableNode& node) {
+ return node.Run(stack_position, stack_position + 1, context);
+ }
+ IterationResult operator()(NullableTerminalNode& node) {
+ return node.Run(*stack_position, context);
+ }
+ IterationResult operator()(AllPresentTerminalNode& node) {
+ return node.Run(*stack_position, context);
+ }
+ IterationResult operator()(AllNullsTerminalNode& node) {
+ return node.Run(*stack_position, context);
+ }
+ template <typename RangeSelector>
+ IterationResult operator()(ListPathNode<RangeSelector>& node) {
+ return node.Run(stack_position, stack_position + 1, context);
+ }
+
+ ElementRange* stack_position;
+ PathWriteContext* context;
+};
+
/// Contains logic for writing a single leaf node to parquet.
/// This tracks the path from root to leaf.
///
@@ -575,32 +624,7 @@ Status WritePath(ElementRange root_range, PathInfo*
path_info,
// |root_range| are processed.
while (stack_position >= stack_base) {
PathInfo::Node& node = path_info->path[stack_position - stack_base];
- struct {
- IterationResult operator()(NullableNode& node) {
- return node.Run(stack_position, stack_position + 1, context);
- }
- IterationResult operator()(ListNode& node) {
- return node.Run(stack_position, stack_position + 1, context);
- }
- IterationResult operator()(NullableTerminalNode& node) {
- return node.Run(*stack_position, context);
- }
- IterationResult operator()(FixedSizeListNode& node) {
- return node.Run(stack_position, stack_position + 1, context);
- }
- IterationResult operator()(AllPresentTerminalNode& node) {
- return node.Run(*stack_position, context);
- }
- IterationResult operator()(AllNullsTerminalNode& node) {
- return node.Run(*stack_position, context);
- }
- IterationResult operator()(LargeListNode& node) {
- return node.Run(stack_position, stack_position + 1, context);
- }
- ElementRange* stack_position;
- PathWriteContext* context;
- } visitor = {stack_position, &context};
-
+ WritePathVisitor visitor = {.stack_position = stack_position, .context =
&context};
IterationResult result = std::visit(visitor, node);
if (ARROW_PREDICT_FALSE(result == kError)) {
@@ -637,20 +661,17 @@ struct FixupVisitor {
int max_rep_level = -1;
int16_t rep_level_if_null = kLevelNotSet;
- template <typename T>
- void HandleListNode(T& arg) {
- if (arg.rep_level() == max_rep_level) {
- arg.SetLast();
+ template <typename RangeSelector>
+ void operator()(ListPathNode<RangeSelector>& node) {
+ if (node.rep_level() == max_rep_level) {
+ node.SetLast();
// after the last list node we don't need to fill
// rep levels on null.
rep_level_if_null = kLevelNotSet;
} else {
- rep_level_if_null = arg.rep_level();
+ rep_level_if_null = node.rep_level();
}
}
- void operator()(ListNode& node) { HandleListNode(node); }
- void operator()(LargeListNode& node) { HandleListNode(node); }
- void operator()(FixedSizeListNode& node) { HandleListNode(node); }
// For non-list intermediate nodes.
template <typename T>
@@ -740,6 +761,23 @@ class PathBuilder {
return VisitInline(*array.values());
}
+ template <typename T>
+ requires ::arrow::is_list_view_type<typename T::TypeClass>::value
+ Status Visit(const T& array) {
+ MaybeAddNullable(array);
+ // Increment necessary due to empty lists.
+ info_.max_def_level++;
+ info_.max_rep_level++;
+ // raw_value_offsets() and raw_value_sizes() account for any slice offset.
+ ListPathNode<ListViewRangeSelector<typename T::offset_type>> node(
+ ListViewRangeSelector<typename
T::offset_type>{array.raw_value_offsets(),
+
array.raw_value_sizes()},
+ info_.max_rep_level, info_.max_def_level - 1);
+ info_.path.emplace_back(std::move(node));
+ nullable_in_parent_ = array.list_view_type()->value_field()->nullable();
+ return VisitInline(*array.values());
+ }
+
Status Visit(const ::arrow::DictionaryArray& array) {
// Only currently handle DictionaryArray where the dictionary is a
// primitive type
@@ -830,8 +868,6 @@ class PathBuilder {
// Types not yet supported in Parquet.
NOT_IMPLEMENTED_VISIT(Union)
NOT_IMPLEMENTED_VISIT(RunEndEncoded);
- NOT_IMPLEMENTED_VISIT(ListView);
- NOT_IMPLEMENTED_VISIT(LargeListView);
#undef NOT_IMPLEMENTED_VISIT
std::vector<PathInfo>& paths() { return paths_; }
diff --git a/cpp/src/parquet/arrow/path_internal_test.cc
b/cpp/src/parquet/arrow/path_internal_test.cc
index 0145e889dd..2daa94e622 100644
--- a/cpp/src/parquet/arrow/path_internal_test.cc
+++ b/cpp/src/parquet/arrow/path_internal_test.cc
@@ -24,6 +24,7 @@
#include <gmock/gmock.h>
#include <gtest/gtest.h>
+#include "arrow/array/array_nested.h"
#include "arrow/testing/gtest_util.h"
#include "arrow/type.h"
@@ -296,6 +297,30 @@ TEST_F(MultipathLevelBuilderTest,
NullableSingleListWithAllPresentEntries) {
EXPECT_THAT(result.post_list_elements[0].end, Eq(3));
}
+TEST_F(MultipathLevelBuilderTest, ListViewOutOfOrder) {
+ auto values = ::arrow::ArrayFromJSON(::arrow::int64(), "[1, 2, 3, 4, 5]");
+ auto offsets = ::arrow::ArrayFromJSON(::arrow::int32(), "[3, 0, 5, 1]");
+ auto sizes = ::arrow::ArrayFromJSON(::arrow::int32(), "[2, 1, 0, 2]");
+ ASSERT_OK_AND_ASSIGN(
+ auto array,
+ ::arrow::ListViewArray::FromArrays(
+ ::arrow::list_view(field("Entries", ::arrow::int64(),
/*nullable=*/false)),
+ *offsets, *sizes, *values, default_memory_pool()));
+
+ ASSERT_OK(
+ MultipathLevelBuilder::Write(*array, /*nullable=*/false, &context_,
callback_));
+
+ ASSERT_THAT(results_, SizeIs(1));
+ const CapturedResult& result = results_[0];
+ result.CheckLevels(/*def_levels=*/{1, 1, 1, 0, 1, 1},
+ /*rep_levels=*/{0, 1, 0, 0, 0, 1});
+ ASSERT_THAT(result.post_list_elements, SizeIs(2));
+ EXPECT_THAT(result.post_list_elements[0].start, Eq(3));
+ EXPECT_THAT(result.post_list_elements[0].end, Eq(5));
+ EXPECT_THAT(result.post_list_elements[1].start, Eq(0));
+ EXPECT_THAT(result.post_list_elements[1].end, Eq(3));
+}
+
TEST_F(MultipathLevelBuilderTest, NullableSingleListWithAllEmptyEntries) {
auto entries = field("Entries", ::arrow::int64(), /*nullable=*/true);
auto list_type = list(entries);
diff --git a/cpp/src/parquet/arrow/reader.cc b/cpp/src/parquet/arrow/reader.cc
index a60af69aec..cc107c1802 100644
--- a/cpp/src/parquet/arrow/reader.cc
+++ b/cpp/src/parquet/arrow/reader.cc
@@ -42,6 +42,7 @@
#include "arrow/util/parallel.h"
#include "arrow/util/range.h"
#include "arrow/util/tracing_internal.h"
+#include "arrow/util/type_traits.h"
#include "parquet/arrow/reader_internal.h"
#include "parquet/bloom_filter.h"
@@ -673,13 +674,45 @@ class ListReader : public ColumnReaderImpl {
const std::shared_ptr<Field> field() override { return field_; }
- private:
+ protected:
std::shared_ptr<ReaderContext> ctx_;
+
+ private:
std::shared_ptr<Field> field_;
::parquet::internal::LevelInfo level_info_;
std::unique_ptr<ColumnReaderImpl> item_reader_;
};
+template <typename IndexType>
+class PARQUET_NO_EXPORT ListViewReader : public ListReader<IndexType> {
+ public:
+ using ListReader<IndexType>::ListReader;
+
+ ::arrow::Result<std::shared_ptr<ChunkedArray>> AssembleArray(
+ std::shared_ptr<ArrayData> data) final {
+ static_assert(::arrow::internal::IsOneOf<IndexType, int32_t,
int64_t>::value);
+ constexpr auto expected_type_id = std::is_same_v<IndexType, int32_t>
+ ? ::arrow::Type::LIST_VIEW
+ : ::arrow::Type::LARGE_LIST_VIEW;
+ DCHECK_EQ(this->field()->type()->id(), expected_type_id);
+ DCHECK_EQ(data->buffers.size(), 2);
+ const auto* offsets = reinterpret_cast<const
IndexType*>(data->buffers[1]->data());
+ ARROW_ASSIGN_OR_RAISE(
+ auto sizes_buffer,
+ AllocateResizableBuffer(sizeof(IndexType) * data->length,
this->ctx_->pool));
+ auto* sizes = reinterpret_cast<IndexType*>(sizes_buffer->mutable_data());
+ for (int64_t i = 0; i < data->length; ++i) {
+ sizes[i] = offsets[i + 1] - offsets[i];
+ }
+ // ListReader produces length + 1 offsets; ListView stores one offset per
slot.
+ data->buffers[1] = ::arrow::SliceBuffer(std::move(data->buffers[1]),
/*offset=*/0,
+ sizeof(IndexType) * data->length);
+ data->buffers.push_back(std::move(sizes_buffer));
+ std::shared_ptr<Array> result = ::arrow::MakeArray(data);
+ return std::make_shared<ChunkedArray>(std::move(result));
+ }
+};
+
class PARQUET_NO_EXPORT FixedSizeListReader : public ListReader<int32_t> {
public:
FixedSizeListReader(std::shared_ptr<ReaderContext> ctx,
std::shared_ptr<Field> field,
@@ -888,7 +921,9 @@ Status GetReader(const SchemaField& field, const
std::shared_ptr<Field>& arrow_f
field.level_info);
} else if (type_id == ::arrow::Type::LIST || type_id == ::arrow::Type::MAP ||
type_id == ::arrow::Type::FIXED_SIZE_LIST ||
- type_id == ::arrow::Type::LARGE_LIST) {
+ type_id == ::arrow::Type::LARGE_LIST ||
+ type_id == ::arrow::Type::LIST_VIEW ||
+ type_id == ::arrow::Type::LARGE_LIST_VIEW) {
auto list_field = arrow_field;
auto child = &field.children[0];
std::unique_ptr<ColumnReaderImpl> child_reader;
@@ -941,6 +976,20 @@ Status GetReader(const SchemaField& field, const
std::shared_ptr<Field>& arrow_f
*out = std::make_unique<ListReader<int64_t>>(ctx, list_field,
field.level_info,
std::move(child_reader));
+ } else if (type_id == ::arrow::Type::LIST_VIEW) {
+ if (!reader_child_type->Equals(schema_child_type)) {
+ list_field =
list_field->WithType(::arrow::list_view(reader_child_type));
+ }
+
+ *out = std::make_unique<ListViewReader<int32_t>>(ctx, list_field,
field.level_info,
+
std::move(child_reader));
+ } else if (type_id == ::arrow::Type::LARGE_LIST_VIEW) {
+ if (!reader_child_type->Equals(schema_child_type)) {
+ list_field =
list_field->WithType(::arrow::large_list_view(reader_child_type));
+ }
+
+ *out = std::make_unique<ListViewReader<int64_t>>(ctx, list_field,
field.level_info,
+
std::move(child_reader));
} else if (type_id == ::arrow::Type::FIXED_SIZE_LIST) {
if (!reader_child_type->Equals(schema_child_type)) {
auto& fixed_list_type =
diff --git a/cpp/src/parquet/arrow/schema.cc b/cpp/src/parquet/arrow/schema.cc
index 9c4c462c6b..bc4de6c39b 100644
--- a/cpp/src/parquet/arrow/schema.cc
+++ b/cpp/src/parquet/arrow/schema.cc
@@ -19,6 +19,7 @@
#include <functional>
#include <string>
+#include <utility>
#include <vector>
#include "arrow/extension/json.h"
@@ -455,7 +456,9 @@ Status FieldToNode(const std::string& name, const
std::shared_ptr<Field>& field,
}
case ArrowTypeId::FIXED_SIZE_LIST:
case ArrowTypeId::LARGE_LIST:
- case ArrowTypeId::LIST: {
+ case ArrowTypeId::LIST:
+ case ArrowTypeId::LARGE_LIST_VIEW:
+ case ArrowTypeId::LIST_VIEW: {
auto list_type =
std::static_pointer_cast<::arrow::BaseListType>(field->type());
return ListToNode(list_type, name, field->nullable(), field_id,
properties,
arrow_properties, out);
@@ -984,6 +987,15 @@ Status GetOriginSchema(const std::shared_ptr<const
KeyValueMetadata>& metadata,
Result<bool> ApplyOriginalMetadata(const Field& origin_field, SchemaField*
inferred);
+template <typename ArrowListType, typename... Args>
+auto GetListFactory(Args&&... args) {
+ return [... args = std::forward<Args>(args)](FieldVector fields) mutable {
+ DCHECK_EQ(fields.size(), 1);
+ return std::make_shared<ArrowListType>(std::move(fields[0]),
+
std::forward<decltype(args)>(args)...);
+ };
+}
+
std::function<std::shared_ptr<::arrow::DataType>(FieldVector)>
GetNestedFactory(
const ArrowType& origin_type, const ArrowType& inferred_type) {
switch (inferred_type.id()) {
@@ -993,28 +1005,26 @@
std::function<std::shared_ptr<::arrow::DataType>(FieldVector)> GetNestedFactory(
}
break;
case ::arrow::Type::LIST:
- case ::arrow::Type::LARGE_LIST:
- if (origin_type.id() == ::arrow::Type::LIST) {
- return [](FieldVector fields) {
- DCHECK_EQ(fields.size(), 1);
- return ::arrow::list(std::move(fields[0]));
- };
- }
- if (origin_type.id() == ::arrow::Type::LARGE_LIST) {
- return [](FieldVector fields) {
- DCHECK_EQ(fields.size(), 1);
- return ::arrow::large_list(std::move(fields[0]));
- };
- }
- if (origin_type.id() == ::arrow::Type::FIXED_SIZE_LIST) {
- const auto list_size =
- checked_cast<const
::arrow::FixedSizeListType&>(origin_type).list_size();
- return [list_size](FieldVector fields) {
- DCHECK_EQ(fields.size(), 1);
- return ::arrow::fixed_size_list(std::move(fields[0]), list_size);
- };
+ case ::arrow::Type::LARGE_LIST: {
+ switch (origin_type.id()) {
+ case ::arrow::Type::LIST:
+ return GetListFactory<::arrow::ListType>();
+ case ::arrow::Type::LARGE_LIST:
+ return GetListFactory<::arrow::LargeListType>();
+ case ::arrow::Type::LIST_VIEW:
+ return GetListFactory<::arrow::ListViewType>();
+ case ::arrow::Type::LARGE_LIST_VIEW:
+ return GetListFactory<::arrow::LargeListViewType>();
+ case ::arrow::Type::FIXED_SIZE_LIST: {
+ const auto list_size =
+ checked_cast<const
::arrow::FixedSizeListType&>(origin_type).list_size();
+ return GetListFactory<::arrow::FixedSizeListType>(list_size);
+ }
+ default:
+ break;
}
break;
+ }
case ::arrow::Type::MAP:
if (origin_type.id() == ::arrow::Type::MAP) {
const bool keys_sorted =
diff --git a/cpp/src/parquet/arrow/writer.cc b/cpp/src/parquet/arrow/writer.cc
index 4b2b06e5e0..e0fbe30821 100644
--- a/cpp/src/parquet/arrow/writer.cc
+++ b/cpp/src/parquet/arrow/writer.cc
@@ -26,6 +26,7 @@
#include <vector>
#include "arrow/array.h"
+#include "arrow/array/concatenate.h"
#include "arrow/extension_type.h"
#include "arrow/ipc/writer.h"
#include "arrow/record_batch.h"
@@ -169,13 +170,24 @@ class ArrowColumnWriterV2 {
leaf_idx, ctx, [&](const MultipathLevelBuilderResult& result) {
size_t visited_component_size =
result.post_list_visited_elements.size();
DCHECK_GT(visited_component_size, 0);
- if (visited_component_size != 1) {
- return Status::NotImplemented(
- "Lists with non-zero length null components are not
supported");
+ std::shared_ptr<Array> values_array;
+ if (visited_component_size == 1) {
+ const ElementRange& range =
result.post_list_visited_elements[0];
+ values_array = result.leaf_array->Slice(range.start,
range.Size());
+ } else {
+ // Multiple leaf ranges can be produced when child values are
+ // skipped, such as null fixed-size-list slots, or when
+ // list-view ranges are non-contiguous. Concatenate the slices
+ // in logical write order.
+ ::arrow::ArrayVector arrays;
+ arrays.reserve(visited_component_size);
+ for (const auto& range : result.post_list_visited_elements) {
+ DCHECK(!range.Empty());
+ arrays.push_back(result.leaf_array->Slice(range.start,
range.Size()));
+ }
+ ARROW_ASSIGN_OR_RAISE(values_array,
+ ::arrow::Concatenate(arrays,
ctx->memory_pool));
}
- const ElementRange& range = result.post_list_visited_elements[0];
- std::shared_ptr<Array> values_array =
- result.leaf_array->Slice(range.start, range.Size());
return column_writer->WriteArrow(result.def_levels,
result.rep_levels,
result.def_rep_level_count,
*values_array,
diff --git a/cpp/src/parquet/properties.h b/cpp/src/parquet/properties.h
index 7eb37c3d52..e2244a1176 100644
--- a/cpp/src/parquet/properties.h
+++ b/cpp/src/parquet/properties.h
@@ -1374,7 +1374,8 @@ class PARQUET_EXPORT ArrowWriterProperties {
/// \brief EXPERIMENTAL: Write binary serialized Arrow schema to the file,
/// to enable certain read options (like "read_dictionary") to be set
- /// automatically
+ /// automatically or read back non-default Arrow types like ListView,
+ /// LargeListView, LargeList, and Arrow extension types.
Builder* store_schema() {
store_schema_ = true;
return this;
diff --git a/docs/source/cpp/parquet.rst b/docs/source/cpp/parquet.rst
index 045f7f80f6..1aace68626 100644
--- a/docs/source/cpp/parquet.rst
+++ b/docs/source/cpp/parquet.rst
@@ -500,7 +500,8 @@ physical type.
* \(3) On the write side, an Arrow Date64 is also mapped to a Parquet DATE
INT32.
-* \(4) On the write side, an Arrow FixedSizedList is also mapped to a Parquet
LIST.
+* \(4) On the write side, an Arrow FixedSizeList, ListView or LargeListView is
+ also mapped to a Parquet LIST.
* \(5) On the read side, a key with multiple values does not get deduplicated,
in contradiction with the
