This is an automated email from the ASF dual-hosted git repository.
pjfanning pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/poi.git
The following commit(s) were added to refs/heads/trunk by this push:
new 76d93fcc0f Spill unknown-size zip entries to temp files only once they
exceed the threshold (#1236)
76d93fcc0f is described below
commit 76d93fcc0fbcd65213898d9dc56fdc60baefabfe
Author: PJ Fanning <[email protected]>
AuthorDate: Tue Sep 1 12:20:24 2026 +0100
Spill unknown-size zip entries to temp files only once they exceed the
threshold (#1236)
When temp-file buffering is enabled via
ZipInputStreamZipEntrySource.setThresholdBytesForTempFiles, every entry
with an unknown declared size (entrySize == -1, e.g. data-descriptor
entries written by streaming zip writers) went straight to a temp file,
however small it actually was - so enabling the threshold cost a temp file
for practically every entry of a streamed zip.
Buffer such entries in memory up to the threshold instead, and spill to a
temp file (or encrypted temp data) only when the entry really turns out to
be that large, logging a warning when that happens. Small unknown-size
entries stay on the heap with no temp file, while an entry that hides its
size behind a data descriptor still cannot force more than the threshold
into memory - e.g. a ~1MB file whose entry inflates to ~92MB while staying
just above the minimum inflate ratio and just under the 100MB per-entry
cap.
Known-size entries behave as before: at or above the threshold they are
copied to a temp file, below it they are read into memory.
Co-authored-by: Claude Opus 4.8 <[email protected]>
---
.../poi/openxml4j/util/ZipArchiveFakeEntry.java | 57 +++++++++++++++---
.../openxml4j/util/TestZipArchiveFakeEntry.java | 68 ++++++++++++++++++++++
2 files changed, 117 insertions(+), 8 deletions(-)
diff --git
a/poi-ooxml/src/main/java/org/apache/poi/openxml4j/util/ZipArchiveFakeEntry.java
b/poi-ooxml/src/main/java/org/apache/poi/openxml4j/util/ZipArchiveFakeEntry.java
index 83c581b9f3..e4a1a85545 100644
---
a/poi-ooxml/src/main/java/org/apache/poi/openxml4j/util/ZipArchiveFakeEntry.java
+++
b/poi-ooxml/src/main/java/org/apache/poi/openxml4j/util/ZipArchiveFakeEntry.java
@@ -22,6 +22,7 @@ import java.nio.file.Files;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.io.input.UnsynchronizedByteArrayInputStream;
+import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream;
import org.apache.logging.log4j.Logger;
import org.apache.poi.logging.PoiLogManager;
import org.apache.poi.poifs.crypt.temp.EncryptedTempData;
@@ -82,17 +83,42 @@ public final class ZipArchiveFakeEntry extends
ZipArchiveEntry implements Closea
if (threshold >= 0 && (entrySize >= threshold || entrySize == -1)) {
boolean success = false;
try {
- if (ZipInputStreamZipEntrySource.shouldEncryptTempFiles()) {
- encryptedTempData = new EncryptedTempData();
- try (OutputStream os =
encryptedTempData.getOutputStream()) {
- numberOfBytes = IOUtils.copy(inp, os);
+ final long bytes;
+ if (entrySize == -1) {
+ // The entry does not declare its uncompressed size (e.g.
it was written
+ // with a data descriptor by a streaming zip writer). Most
such entries
+ // are small, so buffer in memory up to the temp-file
threshold and only
+ // spill to a temp file if the entry really is that large
- an entry that
+ // hides its size cannot force more than the threshold
onto the heap, and
+ // small entries no longer cost a temp file each.
+ try (UnsynchronizedByteArrayOutputStream baos =
UnsynchronizedByteArrayOutputStream.builder().get()) {
+ final long bytesInMemory = IOUtils.copy(inp, baos,
threshold);
+ final int nextByte = bytesInMemory < threshold ? -1 :
inp.read();
+ if (nextByte == -1) {
+ data = baos.toByteArray();
+ bytes = data.length;
+ } else {
+ try (OutputStream os =
createTempDataOutputStream()) {
+ LOG.atWarn().log("Zip entry {} does not
declare its uncompressed size and is larger " +
+ "than the temp-file threshold
of {} bytes - spilling it to {}",
+ entry.getName(), threshold,
+ tempFile != null ?
tempFile.getAbsolutePath() : "encrypted temp data");
+ baos.writeTo(os);
+ os.write(nextByte);
+ bytes = bytesInMemory + 1 + IOUtils.copy(inp,
os);
+ }
+ }
}
} else {
- tempFile = TempFile.createTempFile("poi-zip-entry",
".tmp");
- LOG.atInfo().log("Creating temp file {} for zip entry {}
of size {} bytes",
- tempFile.getAbsolutePath(), entry.getName(),
entrySize);
- numberOfBytes = IOUtils.copy(inp, tempFile);
+ try (OutputStream os = createTempDataOutputStream()) {
+ if (tempFile != null) {
+ LOG.atInfo().log("Creating temp file {} for zip
entry {} of size {} bytes",
+ tempFile.getAbsolutePath(),
entry.getName(), entrySize);
+ }
+ bytes = IOUtils.copy(inp, os);
+ }
}
+ numberOfBytes = bytes;
success = true;
} finally {
if (!success) {
@@ -131,6 +157,21 @@ public final class ZipArchiveFakeEntry extends
ZipArchiveEntry implements Closea
}
}
+ /**
+ * Opens the output to buffer this entry's data outside the heap:
encrypted temp data if
+ * {@link ZipInputStreamZipEntrySource#setEncryptTempFiles(boolean)} is
enabled, a plain
+ * temp file otherwise. Sets the corresponding field so {@link
#getInputStream()} and
+ * {@link #close()} can find it.
+ */
+ private OutputStream createTempDataOutputStream() throws IOException {
+ if (ZipInputStreamZipEntrySource.shouldEncryptTempFiles()) {
+ encryptedTempData = new EncryptedTempData();
+ return encryptedTempData.getOutputStream();
+ }
+ tempFile = TempFile.createTempFile("poi-zip-entry", ".tmp");
+ return Files.newOutputStream(tempFile.toPath());
+ }
+
@Override
public long getSize() {
return numberOfBytes;
diff --git
a/poi-ooxml/src/test/java/org/apache/poi/openxml4j/util/TestZipArchiveFakeEntry.java
b/poi-ooxml/src/test/java/org/apache/poi/openxml4j/util/TestZipArchiveFakeEntry.java
index a3a8e6bddf..68bbf2f342 100644
---
a/poi-ooxml/src/test/java/org/apache/poi/openxml4j/util/TestZipArchiveFakeEntry.java
+++
b/poi-ooxml/src/test/java/org/apache/poi/openxml4j/util/TestZipArchiveFakeEntry.java
@@ -30,6 +30,7 @@ import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -135,6 +136,73 @@ public class TestZipArchiveFakeEntry {
}
}
+ @Test
+ void testUnknownSizeBelowThresholdStaysInMemory() throws IOException {
+ // an entry that does not declare its size (getSize() == -1) is
buffered in memory
+ // as long as it stays below the temp-file threshold - no temp file is
created
+ ZipInputStreamZipEntrySource.setThresholdBytesForTempFiles(1000);
+ try {
+ ZipArchiveEntry entry = new ZipArchiveEntry("hack123");
+ UnsynchronizedByteArrayOutputStream baos =
UnsynchronizedByteArrayOutputStream.builder().get();
+ final byte[] data = "fakeValue".getBytes(StandardCharsets.UTF_8);
+ baos.write(data);
+ try (ZipArchiveFakeEntry zipArchiveFakeEntry = new
ZipArchiveFakeEntry(entry, baos.toInputStream())) {
+ assertFalse(zipArchiveFakeEntry.isUnencryptedTempFileBacked());
+ assertFalse(zipArchiveFakeEntry.isEncryptedTempFileBacked());
+ assertEquals(data.length, zipArchiveFakeEntry.getSize());
+ UnsynchronizedByteArrayOutputStream baos2 =
UnsynchronizedByteArrayOutputStream.builder().get();
+ try (InputStream stream =
zipArchiveFakeEntry.getInputStream()) {
+ IOUtils.copy(stream, baos2);
+ }
+ assertEquals("fakeValue",
baos2.toString(StandardCharsets.UTF_8.name()));
+ }
+ } finally {
+ ZipInputStreamZipEntrySource.setThresholdBytesForTempFiles(-1);
+ }
+ }
+
+ @Test
+ void testUnknownSizeAboveThresholdSpillsToTempFile() throws IOException {
+ // an entry that does not declare its size and turns out to be larger
than the
+ // temp-file threshold spills to a temp file with no data lost
+ ZipInputStreamZipEntrySource.setThresholdBytesForTempFiles(4);
+ try {
+ runTestSpill(true, false);
+ } finally {
+ ZipInputStreamZipEntrySource.setThresholdBytesForTempFiles(-1);
+ }
+ }
+
+ @Test
+ void testUnknownSizeAboveThresholdSpillsToEncryptedTempFile() throws
IOException {
+ ZipInputStreamZipEntrySource.setThresholdBytesForTempFiles(4);
+ ZipInputStreamZipEntrySource.setEncryptTempFiles(true);
+ try {
+ runTestSpill(false, true);
+ } finally {
+ ZipInputStreamZipEntrySource.setThresholdBytesForTempFiles(-1);
+ ZipInputStreamZipEntrySource.setEncryptTempFiles(false);
+ }
+ }
+
+ private static void runTestSpill(boolean unencryptedTempFileExpected,
+ boolean encryptedTempFileExpected) throws
IOException {
+ ZipArchiveEntry entry = new ZipArchiveEntry("hack123");
+ UnsynchronizedByteArrayOutputStream baos =
UnsynchronizedByteArrayOutputStream.builder().get();
+ final byte[] data = "fakeValue".getBytes(StandardCharsets.UTF_8);
+ baos.write(data);
+ try (ZipArchiveFakeEntry zipArchiveFakeEntry = new
ZipArchiveFakeEntry(entry, baos.toInputStream())) {
+ assertEquals(unencryptedTempFileExpected,
zipArchiveFakeEntry.isUnencryptedTempFileBacked());
+ assertEquals(encryptedTempFileExpected,
zipArchiveFakeEntry.isEncryptedTempFileBacked());
+ assertEquals(data.length, zipArchiveFakeEntry.getSize());
+ UnsynchronizedByteArrayOutputStream baos2 =
UnsynchronizedByteArrayOutputStream.builder().get();
+ try (InputStream stream = zipArchiveFakeEntry.getInputStream()) {
+ assertEquals(data.length, IOUtils.copy(stream, baos2));
+ }
+ assertEquals("fakeValue",
baos2.toString(StandardCharsets.UTF_8.name()));
+ }
+ }
+
static void runTestSize(boolean unencryptedTempFileExpected,
boolean encryptedTempFileExpected) throws
IOException {
final String fakeEntryName = "hack123";
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]