This is an automated email from the ASF dual-hosted git repository.
ethanfeng pushed a commit to branch branch-0.6
in repository https://gitbox.apache.org/repos/asf/celeborn.git
The following commit(s) were added to refs/heads/branch-0.6 by this push:
new 610c52b7e [CELEBORN-2128] Close hadoopFs FileSystem when worker is
closed
610c52b7e is described below
commit 610c52b7e9093b33e40e396ec6fe800c37504499
Author: xxx <[email protected]>
AuthorDate: Wed Aug 27 14:12:55 2025 +0800
[CELEBORN-2128] Close hadoopFs FileSystem when worker is closed
### What changes were proposed in this pull request?
Close hadoopFs FileSystem when worker is closed.
### Why are the changes needed?
When the worker is closed, close the hadoopFs FileSystem to avoid resource
leakage.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
CI.
Closes #3449 from xy2953396112/CELEBORN-2128.
Authored-by: xxx <[email protected]>
Signed-off-by: mingji <[email protected]>
(cherry picked from commit d4e13b6ba2e61143ea65b43a3b3d8ef745b0e407)
Signed-off-by: mingji <[email protected]>
---
.../service/deploy/worker/storage/StorageManager.scala | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git
a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala
b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala
index 4d5a35449..7b3f1ed8c 100644
---
a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala
+++
b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala
@@ -808,6 +808,19 @@ final private[worker] class StorageManager(conf:
CelebornConf, workerSource: Abs
}
def close(exitKind: Int): Unit = {
+ if (hadoopFs != null) {
+ hadoopFs.asScala.foreach {
+ case (storageType, fs) =>
+ if (fs != null) {
+ try {
+ fs.close()
+ } catch {
+ case t: Throwable =>
+ logError(s"Close $storageType FileSystem ${fs.getUri}
failed.", t)
+ }
+ }
+ }
+ }
if (db != null) {
if (exitKind == CelebornExitKind.WORKER_GRACEFUL_SHUTDOWN) {
try {
