This is an automated email from the ASF dual-hosted git repository.
marin-ma pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new afaf43fc27 [GLUTEN-13542][VL] Remove an unused arrowGetOrThrow
overload and complex-type flush buffer (#13548)
afaf43fc27 is described below
commit afaf43fc27c6ad08ba423d733c6c287021f9e1e6
Author: YangJie <[email protected]>
AuthorDate: Wed Nov 11 06:11:33 2026 -0500
[GLUTEN-13542][VL] Remove an unused arrowGetOrThrow overload and
complex-type flush buffer (#13548)
---
cpp/core/utils/ArrowStatus.h | 8 --------
cpp/velox/shuffle/VeloxHashShuffleWriter.cc | 26 ++++++--------------------
cpp/velox/shuffle/VeloxHashShuffleWriter.h | 1 -
3 files changed, 6 insertions(+), 29 deletions(-)
diff --git a/cpp/core/utils/ArrowStatus.h b/cpp/core/utils/ArrowStatus.h
index 2660a432c5..21a004228f 100644
--- a/cpp/core/utils/ArrowStatus.h
+++ b/cpp/core/utils/ArrowStatus.h
@@ -38,14 +38,6 @@ inline T arrowGetOrThrow(arrow::Result<T> result) {
return std::move(result).ValueOrDie();
}
-template <typename T>
-inline T arrowGetOrThrow(arrow::Result<T> result, const std::string& message) {
- if (!result.status().ok()) {
- ThrowPendingException(message + " - " + result.status().message());
- }
- return std::move(result).ValueOrDie();
-}
-
static inline void arrowAssertOkOrThrow(arrow::Status status) {
if (!status.ok()) {
throwArrowException(status.message());
diff --git a/cpp/velox/shuffle/VeloxHashShuffleWriter.cc
b/cpp/velox/shuffle/VeloxHashShuffleWriter.cc
index 877b63995e..3fa9166f74 100644
--- a/cpp/velox/shuffle/VeloxHashShuffleWriter.cc
+++ b/cpp/velox/shuffle/VeloxHashShuffleWriter.cc
@@ -229,12 +229,9 @@ arrow::Result<std::shared_ptr<arrow::Buffer>>
VeloxHashShuffleWriter::generateCo
const facebook::velox::IndexRange allRows{0, vector->size()};
serializer->append(vector, folly::Range(&allRows, 1));
auto serializedSize = serializer->maxSerializedSize();
- auto flushBuffer = complexTypeFlushBuffer_[0];
- if (flushBuffer == nullptr) {
- ARROW_ASSIGN_OR_RAISE(flushBuffer,
arrow::AllocateResizableBuffer(serializedSize, partitionBufferPool_.get()));
- } else if (serializedSize > flushBuffer->capacity()) {
- RETURN_NOT_OK(flushBuffer->Reserve(serializedSize));
- }
+ ARROW_ASSIGN_OR_RAISE(
+ std::shared_ptr<arrow::ResizableBuffer> flushBuffer,
+ arrow::AllocateResizableBuffer(serializedSize,
partitionBufferPool_.get()));
auto valueBuffer = arrow::SliceMutableBuffer(flushBuffer, 0, serializedSize);
auto output =
std::make_shared<arrow::io::FixedSizeBufferWriter>(valueBuffer);
@@ -486,13 +483,6 @@ std::vector<int64_t>
VeloxHashShuffleWriter::estimatePartitionBufferBytes() cons
}
}
- for (uint32_t pid = 0; pid < complexTypeFlushBuffer_.size(); ++pid) {
- const auto& buffer = complexTypeFlushBuffer_[pid];
- if (buffer) {
- partitionBytes[pid] += buffer->capacity();
- }
- }
-
return partitionBytes;
}
@@ -865,7 +855,6 @@ arrow::Status VeloxHashShuffleWriter::initColumnTypes(const
facebook::velox::Row
inputHasNull_.resize(simpleColumnIndices_.size(), false);
complexTypeData_.resize(numPartitions_);
- complexTypeFlushBuffer_.resize(numPartitions_);
complexWriteType_ =
std::make_shared<facebook::velox::RowType>(std::move(complexNames),
std::move(complexChildrens));
@@ -1172,13 +1161,10 @@
arrow::Result<std::vector<std::shared_ptr<arrow::Buffer>>> VeloxHashShuffleWrite
}
}
if (hasComplexType_ && complexTypeData_[partitionId] != nullptr) {
- auto flushBuffer = complexTypeFlushBuffer_[partitionId];
auto serializedSize = complexTypeData_[partitionId]->maxSerializedSize();
- if (flushBuffer == nullptr) {
- ARROW_ASSIGN_OR_RAISE(flushBuffer,
arrow::AllocateResizableBuffer(serializedSize, partitionBufferPool_.get()));
- } else if (serializedSize > flushBuffer->capacity()) {
- RETURN_NOT_OK(flushBuffer->Reserve(serializedSize));
- }
+ ARROW_ASSIGN_OR_RAISE(
+ std::shared_ptr<arrow::ResizableBuffer> flushBuffer,
+ arrow::AllocateResizableBuffer(serializedSize,
partitionBufferPool_.get()));
auto valueBuffer = arrow::SliceMutableBuffer(flushBuffer, 0,
serializedSize);
auto output =
std::make_shared<arrow::io::FixedSizeBufferWriter>(valueBuffer);
facebook::velox::serializer::presto::PrestoOutputStreamListener listener;
diff --git a/cpp/velox/shuffle/VeloxHashShuffleWriter.h
b/cpp/velox/shuffle/VeloxHashShuffleWriter.h
index a0cf4a1544..65ec011b70 100644
--- a/cpp/velox/shuffle/VeloxHashShuffleWriter.h
+++ b/cpp/velox/shuffle/VeloxHashShuffleWriter.h
@@ -508,7 +508,6 @@ class VeloxHashShuffleWriter : public VeloxShuffleWriter {
// Used by complex types.
// Partition id -> Serialized complex data.
std::vector<std::unique_ptr<facebook::velox::IterativeVectorSerializer>>
complexTypeData_;
- std::vector<std::shared_ptr<arrow::ResizableBuffer>> complexTypeFlushBuffer_;
std::shared_ptr<const facebook::velox::RowType> complexWriteType_;
facebook::velox::serializer::presto::PrestoVectorSerde serde_;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]