This is an automated email from the ASF dual-hosted git repository.
felixybw pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 7e35388d3f [VL] Ignore null values when copy velox string buffer to
arrow (#11944)
7e35388d3f is described below
commit 7e35388d3f3032c932c8caafe10c81fb9f6939ed
Author: wankun <[email protected]>
AuthorDate: Fri Apr 17 08:54:57 2026 +0800
[VL] Ignore null values when copy velox string buffer to arrow (#11944)
Bug fix for VeloxHashShuffleWriter.
Description:
VeloxHashShuffleWriter::write() method will try to convert the input velox
flat vectors into Arrow buffer.
For StringView vectors, VeloxHashShuffleWriter calculates the total size of
all input StringViews, and initializes the Arrow buffer accordingly.
For null values, we do not actually need to copy the dirty values into
Arrow buffers, thereby saving execution memory.
---
cpp/velox/shuffle/VeloxHashShuffleWriter.cc | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/cpp/velox/shuffle/VeloxHashShuffleWriter.cc
b/cpp/velox/shuffle/VeloxHashShuffleWriter.cc
index f57a0a624d..6024f04ab1 100644
--- a/cpp/velox/shuffle/VeloxHashShuffleWriter.cc
+++ b/cpp/velox/shuffle/VeloxHashShuffleWriter.cc
@@ -111,7 +111,7 @@ arrow::Status collectFlatVectorBufferStringView(
auto* rawLength =
reinterpret_cast<gluten::StringLengthType*>(lengthBuffer->mutable_data());
uint64_t offset = 0;
for (int32_t i = 0; i < flatVector->size(); i++) {
- auto length = rawValues[i].size();
+ auto length = flatVector->isNullAt(i) ? 0 : rawValues[i].size();
*rawLength++ = length;
offset += length;
}
@@ -120,6 +120,9 @@ arrow::Status collectFlatVectorBufferStringView(
ARROW_ASSIGN_OR_RAISE(auto valueBuffer,
arrow::AllocateResizableBuffer(offset, pool));
auto raw = reinterpret_cast<char*>(valueBuffer->mutable_data());
for (int32_t i = 0; i < flatVector->size(); i++) {
+ if (flatVector->isNullAt(i)) {
+ continue;
+ }
gluten::fastCopy(raw, rawValues[i].data(), rawValues[i].size());
raw += rawValues[i].size();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
