This is an automated email from the ASF dual-hosted git repository.
hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new c571c20941 [VL] Delta write / Spark 3.2, 3.3 write: Fix null sink
memory pool causing crash with remote file systems (#11410)
c571c20941 is described below
commit c571c20941a97c624c6971b4eefde9b132bfcc65
Author: Hongze Zhang <[email protected]>
AuthorDate: Thu Jan 15 11:08:08 2026 +0000
[VL] Delta write / Spark 3.2, 3.3 write: Fix null sink memory pool causing
crash with remote file systems (#11410)
Fixes the following error:
```
facebook::velox::filesystems::S3WriteFile::Impl::Impl(std::basic_string_view<char,
std::char_traits<char> >, Aws::S3::S3Client*,
facebook::velox::memory::MemoryPool*)+0xb6)[0x7f9892b99726]
facebook::velox::filesystems::S3WriteFile::S3WriteFile(std::basic_string_view<char,
std::char_traits<char> >, Aws::S3::S3Client*,
facebook::velox::memory::MemoryPool*)+0x81)[0x7f9892b973e1]
facebook::velox::filesystems::S3FileSystem::openFileForWrite(std::basic_string_view<char,
std::char_traits<char> >, facebook::velox::filesystems::FileOptions
const&)+0x54)[0x7f9892b85e64]
facebook::velox::filesystems::s3WriteFileSinkGenerator(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
facebook::velox::dwio::common::FileSink::Options const&)+0x155)[0x7f9892b83335]
std::_Function_handler<std::unique_ptr<facebook::velox::dwio::common::FileSink,
std::default_delete<facebook::velox::dwio::common::FileSink> >
(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>
> const&, facebook::velox::dwio::common::FileSink::Options const&),
std::unique_ptr<facebook::velox::dwio::common::FileSink,
std::default_delete<facebook::velox::dwio::common::FileSink> >
(*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<cha
[...]
facebook::velox::dwio::common::FileSink::create(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
facebook::velox::dwio::common::FileSink::Options const&)+0x5f)[0x7f9891b668bf]
gluten::VeloxParquetDataSourceS3::initSink(std::unordered_map<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> >,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>
>, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > >, std::equal_to<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > >,
std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<
[...]
gluten::VeloxParquetDataSource::init(std::unordered_map<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> >,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>
>, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > >, std::equal_to<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > >,
std::allocator<std::pair<std::__cxx11::basic_string<char,
std::char_traits<char>, [...]
Java_org_apache_gluten_datasource_VeloxDataSourceJniWrapper_init+0x370)[0x7f98916f5d70]
```
---
cpp/velox/operators/writer/VeloxParquetDataSource.cc | 2 +-
cpp/velox/operators/writer/VeloxParquetDataSource.h | 6 +++++-
2 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/cpp/velox/operators/writer/VeloxParquetDataSource.cc
b/cpp/velox/operators/writer/VeloxParquetDataSource.cc
index cd5063a61d..5e9f425136 100644
--- a/cpp/velox/operators/writer/VeloxParquetDataSource.cc
+++ b/cpp/velox/operators/writer/VeloxParquetDataSource.cc
@@ -36,7 +36,7 @@ namespace gluten {
void VeloxParquetDataSource::initSink(const std::unordered_map<std::string,
std::string>& /* sparkConfs */) {
if (strncmp(filePath_.c_str(), "file:", 5) == 0) {
- sink_ = dwio::common::FileSink::create(filePath_, {.pool = pool_.get()});
+ sink_ = dwio::common::FileSink::create(filePath_, {.pool =
sinkPool_.get()});
} else {
throw std::runtime_error("The file path is not local when writing data
with parquet format in velox runtime!");
}
diff --git a/cpp/velox/operators/writer/VeloxParquetDataSource.h
b/cpp/velox/operators/writer/VeloxParquetDataSource.h
index 4dde5c69aa..dac97168d5 100644
--- a/cpp/velox/operators/writer/VeloxParquetDataSource.h
+++ b/cpp/velox/operators/writer/VeloxParquetDataSource.h
@@ -84,7 +84,11 @@ class VeloxParquetDataSource : public VeloxDataSource {
std::shared_ptr<facebook::velox::memory::MemoryPool> veloxPool,
std::shared_ptr<facebook::velox::memory::MemoryPool> sinkPool,
std::shared_ptr<arrow::Schema> schema)
- : VeloxDataSource(filePath, schema), filePath_(filePath),
schema_(schema), pool_(std::move(veloxPool)) {}
+ : VeloxDataSource(filePath, schema),
+ filePath_(filePath),
+ sinkPool_(std::move(sinkPool)),
+ schema_(schema),
+ pool_(std::move(veloxPool)) {}
void init(const std::unordered_map<std::string, std::string>& sparkConfs)
override;
virtual void initSink(const std::unordered_map<std::string, std::string>&
sparkConfs);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
