This is an automated email from the ASF dual-hosted git repository. chibenwa pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/james-mime4j.git
commit 46b4d91a3ec6fdd0ab038cd5bead848ab5ec18bc Author: Benoit TELLIER <[email protected]> AuthorDate: Tue Nov 10 10:53:19 2026 +0100 [FIX] TempFileStorageProvider: a deleted temporary file leaves nothing registered Each spooled part called File.deleteOnExit(), which adds its path to the static set of java.io.DeleteOnExitHook. That set is read only at shutdown and never shrinks, even once TempFileStorage.delete() removed the file. As the default provider spools every part of 1 KB or more, a long-running service parsing mail through StorageBodyFactory kept one path per part in the heap until the JVM stopped: a sender sending many messages of small parts, or ordinary traffic over time, made the heap grow without bound. Files that could not be deleted also stayed in an unbounded static set, retried by every later delete() under one global lock. The provider now keeps the files not deleted yet in a concurrent set that delete() removes them from, and one shutdown hook deletes whatever is left. Failed deletions are retried by later deletions without a global lock, up to MAX_PENDING_DELETIONS files; the others are left to the shutdown hook. --- .../mime4j/storage/TempFileStorageProvider.java | 112 ++++++++++++++++----- .../TempFileStorageProviderCleanupTest.java | 88 ++++++++++++++++ 2 files changed, 173 insertions(+), 27 deletions(-) diff --git a/storage/src/main/java/org/apache/james/mime4j/storage/TempFileStorageProvider.java b/storage/src/main/java/org/apache/james/mime4j/storage/TempFileStorageProvider.java index 0e87f928..a9a5958c 100644 --- a/storage/src/main/java/org/apache/james/mime4j/storage/TempFileStorageProvider.java +++ b/storage/src/main/java/org/apache/james/mime4j/storage/TempFileStorageProvider.java @@ -27,10 +27,9 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Files; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Optional; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.io.FileUtils; @@ -51,6 +50,23 @@ public class TempFileStorageProvider extends AbstractStorageProvider { private static final String DEFAULT_PREFIX = "m4j"; + /** + * Bound on the files whose deletion failed and is retried by later deletions; the files + * beyond it are only deleted at shutdown. + */ + static final int MAX_PENDING_DELETIONS = 1024; + + /** + * The temporary files created and not deleted yet. Unlike {@link File#deleteOnExit()}, whose + * registrations stay in memory until the JVM stops, a file leaves this set once deleted. + */ + private static final Set<File> LIVE_FILES = ConcurrentHashMap.newKeySet(); + + /** The files whose deletion failed, retried by later deletions. */ + private static final Set<File> PENDING_DELETIONS = ConcurrentHashMap.newKeySet(); + + private static final AtomicBoolean SHUTDOWN_HOOK_REGISTERED = new AtomicBoolean(); + private final String prefix; private final String suffix; private final File directory; @@ -115,11 +131,56 @@ public class TempFileStorageProvider extends AbstractStorageProvider { } else { file = Files.createTempFile(directory.toPath(), prefix, suffix).toFile(); } - file.deleteOnExit(); + register(file); return new TempFileStorageOutputStream(file); } + private static void register(File file) { + if (SHUTDOWN_HOOK_REGISTERED.compareAndSet(false, true)) { + try { + Runtime.getRuntime().addShutdownHook( + new Thread(TempFileStorageProvider::deleteLiveFiles, "mime4j-temp-file-cleanup")); + } catch (IllegalStateException | SecurityException e) { + // shutting down, or not allowed: the files are only deleted by their storage + } + } + LIVE_FILES.add(file); + } + + private static void deleteLiveFiles() { + for (File file : LIVE_FILES) { + tryDelete(file); + } + } + + /** + * Deletes the given file. + * + * @return whether the file no longer exists + */ + private static boolean tryDelete(File file) { + if (file.delete() || !file.exists()) { + LIVE_FILES.remove(file); + return true; + } + return false; + } + + /** + * Number of temporary files created and not deleted yet. Package private, for tests. + */ + static int liveFileCount() { + return LIVE_FILES.size(); + } + + /** + * Whether the given temporary file is still registered for deletion. Package private, for tests. + */ + static boolean isRegistered(File file) { + return LIVE_FILES.contains(file); + } + private static final class TempFileStorageOutputStream extends StorageOutputStream { private final File file; @@ -151,9 +212,7 @@ public class TempFileStorageProvider extends AbstractStorageProvider { private static final class TempFileStorage implements Storage { - private File file; - - private static final Set<File> filesToDelete = new HashSet<File>(); + private volatile File file; public TempFileStorage(File file) { this.file = file; @@ -162,34 +221,33 @@ public class TempFileStorageProvider extends AbstractStorageProvider { public void delete() { // deleting a file might not immediately succeed if there are still // streams left open (especially under Windows). so we keep track of - // the files that have to be deleted and try to delete all these - // files each time this method gets invoked. - - // a better but more complicated solution would be to start a - // separate thread that tries to delete the files periodically. - - synchronized (filesToDelete) { - if (file != null) { - filesToDelete.add(file); - file = null; - } + // the files that could not be deleted (up to a bound) and try to + // delete them again each time this method gets invoked; whatever + // is left is deleted at shutdown. + File toDelete; + synchronized (this) { + toDelete = file; + file = null; + } + if (toDelete != null && !tryDelete(toDelete) + && PENDING_DELETIONS.size() < MAX_PENDING_DELETIONS) { + PENDING_DELETIONS.add(toDelete); + } - for (Iterator<File> iterator = filesToDelete.iterator(); iterator - .hasNext();) { - try { - FileUtils.forceDelete(iterator.next()); - iterator.remove(); - } catch (IOException e) { - } + for (File pending : PENDING_DELETIONS) { + // whoever removes it from the set retries it + if (pending != toDelete && PENDING_DELETIONS.remove(pending) && !tryDelete(pending)) { + PENDING_DELETIONS.add(pending); } } } public InputStream getInputStream() throws IOException { - if (file == null) + File current = file; + if (current == null) throw new IllegalStateException("storage has been deleted"); - return new BufferedInputStream(new FileInputStream(file)); + return new BufferedInputStream(new FileInputStream(current)); } } diff --git a/storage/src/test/java/org/apache/james/mime4j/storage/TempFileStorageProviderCleanupTest.java b/storage/src/test/java/org/apache/james/mime4j/storage/TempFileStorageProviderCleanupTest.java new file mode 100644 index 00000000..650c5600 --- /dev/null +++ b/storage/src/test/java/org/apache/james/mime4j/storage/TempFileStorageProviderCleanupTest.java @@ -0,0 +1,88 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.mime4j.storage; + +import java.io.File; +import java.lang.reflect.Field; + +import org.junit.Assert; +import org.junit.Test; + +/** + * A deleted temporary storage leaves nothing registered behind it, unlike + * {@link File#deleteOnExit()} whose registrations stay in memory until the JVM stops. + */ +public class TempFileStorageProviderCleanupTest { + + private static File fileOf(Storage storage) throws Exception { + Field field = storage.getClass().getDeclaredField("file"); + field.setAccessible(true); + return (File) field.get(storage); + } + + @Test + public void deletedStoragesShouldLeaveNothingRegistered() throws Exception { + TempFileStorageProvider provider = new TempFileStorageProvider(); + int before = TempFileStorageProvider.liveFileCount(); + for (int i = 0; i < 1000; i++) { + StorageOutputStream out = provider.createStorageOutputStream(); + out.write(new byte[] {1, 2, 3}); + Storage storage = out.toStorage(); + File file = fileOf(storage); + Assert.assertTrue(file.exists()); + Assert.assertTrue(TempFileStorageProvider.isRegistered(file)); + + storage.delete(); + + Assert.assertFalse(file.exists()); + Assert.assertFalse(TempFileStorageProvider.isRegistered(file)); + } + Assert.assertEquals(before, TempFileStorageProvider.liveFileCount()); + } + + @Test + public void deletingTwiceShouldBeHarmless() throws Exception { + StorageOutputStream out = new TempFileStorageProvider().createStorageOutputStream(); + out.write(new byte[] {1}); + Storage storage = out.toStorage(); + File file = fileOf(storage); + storage.delete(); + storage.delete(); + Assert.assertFalse(file.exists()); + Assert.assertFalse(TempFileStorageProvider.isRegistered(file)); + try { + storage.getInputStream(); + Assert.fail("Expected IllegalStateException"); + } catch (IllegalStateException expected) { + // expected + } + } + + @Test + public void aFileRemovedBehindTheStorageShouldStillBeUnregistered() throws Exception { + StorageOutputStream out = new TempFileStorageProvider().createStorageOutputStream(); + out.write(new byte[] {1}); + Storage storage = out.toStorage(); + File file = fileOf(storage); + Assert.assertTrue(file.delete()); + storage.delete(); + Assert.assertFalse(TempFileStorageProvider.isRegistered(file)); + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
