rok commented on code in PR #55251:
URL: https://github.com/apache/arrow/pull/55251#discussion_r4607977715


##########
cpp/src/parquet/arrow/writer.cc:
##########
@@ -135,6 +135,42 @@ Status GetSchemaMetadata(const ::arrow::Schema& schema, 
::arrow::MemoryPool* poo
   return Status::OK();
 }
 
+// Check that all chunks contain supported fixed-size lists with no missing 
values
+bool CanWriteFixedSizeLists(const std::vector<std::shared_ptr<Array>>& arrays) 
{
+  if (arrays.empty() || arrays.front()->type_id() != 
::arrow::Type::FIXED_SIZE_LIST) {
+    return false;
+  }
+  const auto& type =
+      checked_cast<const ::arrow::FixedSizeListType&>(*arrays.front()->type());
+  if (type.list_size() <= 0) {
+    return false;
+  }
+  switch (type.value_type()->id()) {
+    case ::arrow::Type::INT32:
+    case ::arrow::Type::INT64:
+    case ::arrow::Type::FLOAT:
+    case ::arrow::Type::DOUBLE:
+      break;
+    default:
+      return false;
+  }

Review Comment:
   Is there a reason to limit this to these four types? We have more fixed size 
length types. Howw about something like:
   ```cpp
   const auto& value_type = *type.value_type();
   if (::arrow::is_nested(value_type.id()) ||
       ::arrow::is_base_binary_like(value_type.id()) ||
       ::arrow::is_binary_view_like(value_type.id()) ||
       value_type.id() == ::arrow::Type::DICTIONARY ||
       value_type.id() == ::arrow::Type::EXTENSION) {
     return false;
   }
   ```



##########
cpp/src/arrow/util/rle_encoding_internal.h:
##########
@@ -1526,6 +1529,28 @@ inline bool RleBitPackedEncoder::Put(uint64_t value) {
   return true;
 }
 
+inline int RleBitPackedEncoder::PutRepeated(uint64_t value, int count) {
+  ARROW_DCHECK_GE(count, 0);
+  ARROW_DCHECK(bit_width_ == 64 || value < (uint64_t{1} << bit_width_));

Review Comment:
   For consistency, see line 1502 above.
   ```suggestion
     ARROW_DCHECK(bit_width_ == 64 || value < (1ULL << bit_width_));
   ```



##########
cpp/src/arrow/util/rle_encoding_internal.h:
##########
@@ -1526,6 +1529,28 @@ inline bool RleBitPackedEncoder::Put(uint64_t value) {
   return true;
 }
 
+inline int RleBitPackedEncoder::PutRepeated(uint64_t value, int count) {
+  ARROW_DCHECK_GE(count, 0);
+  ARROW_DCHECK(bit_width_ == 64 || value < (uint64_t{1} << bit_width_));
+
+  int encoded = 0;
+  while (encoded < count && !buffer_full_) {
+    if (current_value_ == value && repeat_count_ > 8) {

Review Comment:
   You could flatten the while condition and move things around some more
   ```cpp
     while (encoded < count && !(current_value_ == value && repeat_count_ > 8)) 
{
   ```



##########
cpp/src/parquet/arrow/writer.cc:
##########
@@ -165,6 +201,30 @@ class ArrowColumnWriterV2 {
       } else {
         PARQUET_CATCH_NOT_OK(column_writer = row_group_writer_->NextColumn());
       }
+      if (!fixed_size_lists_.empty()) {
+        if (column_writer->properties()->content_defined_chunking_enabled()) {
+          for (const auto& array : fixed_size_lists_) {
+            ARROW_ASSIGN_OR_RAISE(auto builder,
+                                  MultipathLevelBuilder::Make(*array, 
is_nullable_));
+            level_builders_.emplace_back(std::move(builder));
+          }
+          fixed_size_lists_.clear();
+        } else {
+          const auto& type = checked_cast<const ::arrow::FixedSizeListType&>(
+              *fixed_size_lists_.front()->type());
+          auto fast_ctx = *ctx;
+          fast_ctx.fixed_size_list_size = type.list_size();

Review Comment:
   Adding list size to `ArrowWriteContext` is not ideal. Can we pass it in 
another way?



##########
cpp/src/parquet/arrow/writer.cc:
##########
@@ -298,15 +353,33 @@ class ArrowColumnWriterV2 {
       }
       values_written += chunk_write_size;
     }
-    return std::make_unique<ArrowColumnWriterV2>(std::move(builders), 
column_index,
-                                                 leaf_count, row_group_writer);
+    // Check every chunk before choosing one writing path for the column

Review Comment:
   Please add a comment explaining why buffered row groups are excluded.



##########
cpp/src/parquet/arrow/writer.cc:
##########
@@ -135,6 +135,42 @@ Status GetSchemaMetadata(const ::arrow::Schema& schema, 
::arrow::MemoryPool* poo
   return Status::OK();
 }
 
+// Check that all chunks contain supported fixed-size lists with no missing 
values
+bool CanWriteFixedSizeLists(const std::vector<std::shared_ptr<Array>>& arrays) 
{
+  if (arrays.empty() || arrays.front()->type_id() != 
::arrow::Type::FIXED_SIZE_LIST) {

Review Comment:
   This rejects extension arrays whose storage type is fixed-size. The generic 
path would unwrap extension arrays via `array.storage()` in `path_internal.cc`.



##########
cpp/src/parquet/arrow/writer.cc:
##########
@@ -165,6 +201,30 @@ class ArrowColumnWriterV2 {
       } else {
         PARQUET_CATCH_NOT_OK(column_writer = row_group_writer_->NextColumn());
       }
+      if (!fixed_size_lists_.empty()) {
+        if (column_writer->properties()->content_defined_chunking_enabled()) {
+          for (const auto& array : fixed_size_lists_) {
+            ARROW_ASSIGN_OR_RAISE(auto builder,
+                                  MultipathLevelBuilder::Make(*array, 
is_nullable_));
+            level_builders_.emplace_back(std::move(builder));
+          }
+          fixed_size_lists_.clear();
+        } else {
+          const auto& type = checked_cast<const ::arrow::FixedSizeListType&>(
+              *fixed_size_lists_.front()->type());
+          auto fast_ctx = *ctx;
+          fast_ctx.fixed_size_list_size = type.list_size();
+          for (const auto& array : fixed_size_lists_) {
+            const auto& lists = checked_cast<const 
::arrow::FixedSizeListArray&>(*array);
+            // Take only the numbers belonging to this slice of lists

Review Comment:
   nit
   ```suggestion
               // Take only the values belonging to this slice of lists
   ```



##########
cpp/src/parquet/arrow/writer.cc:
##########
@@ -135,6 +135,42 @@ Status GetSchemaMetadata(const ::arrow::Schema& schema, 
::arrow::MemoryPool* poo
   return Status::OK();
 }
 
+// Check that all chunks contain supported fixed-size lists with no missing 
values
+bool CanWriteFixedSizeLists(const std::vector<std::shared_ptr<Array>>& arrays) 
{
+  if (arrays.empty() || arrays.front()->type_id() != 
::arrow::Type::FIXED_SIZE_LIST) {
+    return false;
+  }
+  const auto& type =
+      checked_cast<const ::arrow::FixedSizeListType&>(*arrays.front()->type());
+  if (type.list_size() <= 0) {
+    return false;
+  }
+  switch (type.value_type()->id()) {
+    case ::arrow::Type::INT32:
+    case ::arrow::Type::INT64:
+    case ::arrow::Type::FLOAT:
+    case ::arrow::Type::DOUBLE:
+      break;
+    default:
+      return false;
+  }
+  for (const auto& array : arrays) {
+    if (!array->type()->Equals(arrays.front()->type())) {
+      return false;
+    }

Review Comment:
   IIUC these arrays come from a chunked array and we can expect them to be of 
the same type. Please double check me.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to