This is an automated email from the ASF dual-hosted git repository.
kejia pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 333c393eb2 [GLUTEN-8072][VL] Call HdfsFileSystem#close() method to
close the hdfs connection prior to the jvm termination (#10593)
333c393eb2 is described below
commit 333c393eb2fb525c1e434598f7c25bb7e3850317
Author: JiaKe <[email protected]>
AuthorDate: Mon Sep 22 14:54:50 2025 +0800
[GLUTEN-8072][VL] Call HdfsFileSystem#close() method to close the hdfs
connection prior to the jvm termination (#10593)
---
cpp/velox/compute/VeloxBackend.cc | 28 ++++++++++++++++++++++
cpp/velox/compute/VeloxBackend.h | 20 +---------------
.../operators/writer/VeloxParquetDataSource.h | 1 -
3 files changed, 29 insertions(+), 20 deletions(-)
diff --git a/cpp/velox/compute/VeloxBackend.cc
b/cpp/velox/compute/VeloxBackend.cc
index 04faab8f3a..e26cd852b6 100644
--- a/cpp/velox/compute/VeloxBackend.cc
+++ b/cpp/velox/compute/VeloxBackend.cc
@@ -44,6 +44,7 @@
#include "velox/connectors/hive/HiveDataSource.h"
#include
"velox/connectors/hive/storage_adapters/abfs/RegisterAbfsFileSystem.h" //
@manual
#include "velox/connectors/hive/storage_adapters/gcs/RegisterGcsFileSystem.h"
// @manual
+#include "velox/connectors/hive/storage_adapters/hdfs/HdfsFileSystem.h"
#include
"velox/connectors/hive/storage_adapters/hdfs/RegisterHdfsFileSystem.h" //
@manual
#include "velox/connectors/hive/storage_adapters/s3fs/RegisterS3FileSystem.h"
// @manual
#include "velox/dwio/orc/reader/OrcReader.h"
@@ -331,4 +332,31 @@ VeloxBackend* VeloxBackend::get() {
}
return instance_.get();
}
+
+void VeloxBackend::tearDown() {
+#ifdef ENABLE_HDFS
+ for (const auto& [_, filesystem] :
facebook::velox::filesystems::registeredFilesystems) {
+ filesystem->close();
+ }
+#endif
+
+ // Destruct IOThreadPoolExecutor will join all threads.
+ // On threads exit, thread local variables can be constructed with
referencing global variables.
+ // So, we need to destruct IOThreadPoolExecutor and stop the threads before
global variables get destructed.
+ ioExecutor_.reset();
+ globalMemoryManager_.reset();
+
+ // dump cache stats on exit if enabled
+ if
(dynamic_cast<facebook::velox::cache::AsyncDataCache*>(asyncDataCache_.get())) {
+ LOG(INFO) << asyncDataCache_->toString();
+ for (const auto& entry :
std::filesystem::directory_iterator(cachePathPrefix_)) {
+ if (entry.path().filename().string().find(cacheFilePrefix_) !=
std::string::npos) {
+ LOG(INFO) << "Removing cache file " <<
entry.path().filename().string();
+ std::filesystem::remove(cachePathPrefix_ + "/" +
entry.path().filename().string());
+ }
+ }
+ asyncDataCache_->shutdown();
+ }
+}
+
} // namespace gluten
diff --git a/cpp/velox/compute/VeloxBackend.h b/cpp/velox/compute/VeloxBackend.h
index be923c22d1..94e7ec93fb 100644
--- a/cpp/velox/compute/VeloxBackend.h
+++ b/cpp/velox/compute/VeloxBackend.h
@@ -56,25 +56,7 @@ class VeloxBackend {
return globalMemoryManager_.get();
}
- void tearDown() {
- // Destruct IOThreadPoolExecutor will join all threads.
- // On threads exit, thread local variables can be constructed with
referencing global variables.
- // So, we need to destruct IOThreadPoolExecutor and stop the threads
before global variables get destructed.
- ioExecutor_.reset();
- globalMemoryManager_.reset();
-
- // dump cache stats on exit if enabled
- if
(dynamic_cast<facebook::velox::cache::AsyncDataCache*>(asyncDataCache_.get())) {
- LOG(INFO) << asyncDataCache_->toString();
- for (const auto& entry :
std::filesystem::directory_iterator(cachePathPrefix_)) {
- if (entry.path().filename().string().find(cacheFilePrefix_) !=
std::string::npos) {
- LOG(INFO) << "Removing cache file " <<
entry.path().filename().string();
- std::filesystem::remove(cachePathPrefix_ + "/" +
entry.path().filename().string());
- }
- }
- asyncDataCache_->shutdown();
- }
- }
+ void tearDown();
private:
explicit VeloxBackend(
diff --git a/cpp/velox/operators/writer/VeloxParquetDataSource.h
b/cpp/velox/operators/writer/VeloxParquetDataSource.h
index 8bbbd5ef06..4dde5c69aa 100644
--- a/cpp/velox/operators/writer/VeloxParquetDataSource.h
+++ b/cpp/velox/operators/writer/VeloxParquetDataSource.h
@@ -39,7 +39,6 @@
#include "velox/connectors/hive/storage_adapters/gcs/GcsFileSystem.h"
#endif
#ifdef ENABLE_HDFS
-#include "velox/connectors/hive/storage_adapters/hdfs/HdfsFileSystem.h"
#include "velox/connectors/hive/storage_adapters/hdfs/HdfsUtil.h"
#endif
#ifdef ENABLE_ABFS
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
