This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 204ddced4e TIKA-5043: Let RarParser read RAR5 archives (#3536)
204ddced4e is described below
commit 204ddced4eab54a02d6c401c03e441276c8ff085
Author: Dominik Schmidt <[email protected]>
AuthorDate: Thu Nov 5 21:38:18 2026 +0100
TIKA-5043: Let RarParser read RAR5 archives (#3536)
* TIKA-5043: let RarParser read RAR5 archives
* TIKA-5043: RAR5 times as instants, cap the dictionary size
RAR5 stores times as Unix time or Windows FILETIME, instants, where RAR4
stores DOS local time; they are now set as UTC dates instead of the JVM zone's
wall clock. junrar allows a 4 GB dictionary by default and a RAR5 window grows
with what is unpacked; archives are opened with a 1 GB cap, what WinRAR 5 and 6
write. A test writes an mtime into the synthetic RAR5 and parses it under three
time zones.
---
CHANGES.txt | 4 +
.../java/org/apache/tika/parser/pkg/RarParser.java | 33 +++++---
.../org/apache/tika/parser/pkg/RarParserTest.java | 94 ++++++++++++++++++++++
3 files changed, 119 insertions(+), 12 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index ccb3c9f882..3e090af93b 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -81,6 +81,10 @@ Release 4.3.0 - ???
EXTRACTION AND FORMATS
+ * RAR5 archives are parsed: RarParser no longer refuses them, junrar has
read RAR5
+ since 8.0.0 (TIKA-5043).
+
+
* Jupyter notebooks (.ipynb, new type application/x-ipynb+json) are parsed:
cell
sources and plain-text outputs as text, kernel, language and format
version as
jupyter:kernel, jupyter:language and jupyter:nbformat, and the image
outputs as
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/RarParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/RarParser.java
index 377667da03..c121232047 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/RarParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/RarParser.java
@@ -22,6 +22,8 @@ import java.util.Collections;
import java.util.Set;
import com.github.junrar.Archive;
+import com.github.junrar.ArchiveOptions;
+import com.github.junrar.RarFormat;
import com.github.junrar.exception.RarException;
import com.github.junrar.rarfile.FileHeader;
import org.apache.commons.io.input.ProxyInputStream;
@@ -31,13 +33,11 @@ import org.xml.sax.SAXException;
import org.apache.tika.annotation.TikaComponent;
import org.apache.tika.exception.EncryptedDocumentException;
import org.apache.tika.exception.TikaException;
-import org.apache.tika.exception.UnsupportedFormatException;
import org.apache.tika.extractor.EmbeddedDocumentExtractor;
import org.apache.tika.extractor.EmbeddedDocumentUtil;
import org.apache.tika.io.TemporaryResources;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.FileSystem;
-import org.apache.tika.metadata.HttpHeaders;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.ParseContext;
@@ -51,6 +51,12 @@ import org.apache.tika.sax.XHTMLContentHandler;
public class RarParser implements Parser {
private static final long serialVersionUID = 6157727985054451501L;
+ /**
+ * The largest dictionary an archive may declare. WinRAR 5 and 6 write up
to 1 GB; junrar's
+ * default is 4 GB, and a RAR5 window grows with the data unpacked into it.
+ */
+ static final long MAX_DICTIONARY_SIZE = 1024L * 1024 * 1024;
+
private static final Set<MediaType> SUPPORTED_TYPES =
Collections.singleton(MediaType.application("x-rar-compressed"));
@@ -68,14 +74,10 @@ public class RarParser implements Parser {
EmbeddedDocumentExtractor extractor =
EmbeddedDocumentUtil.getEmbeddedDocumentExtractor(context);
- String mediaType = metadata.get(HttpHeaders.CONTENT_TYPE);
-
- if (mediaType != null && mediaType.contains("version=5")) {
- throw new UnsupportedFormatException("Tika does not yet support
rar version 5.");
- }
Archive rar = null;
try (TemporaryResources tmp = new TemporaryResources()) {
- rar = new Archive(tis.getFile());
+ rar = new Archive(tis.getFile(),
+
ArchiveOptions.builder().maxDictionarySize(MAX_DICTIONARY_SIZE).build());
if (rar.isEncrypted()) {
throw new EncryptedDocumentException();
@@ -89,10 +91,17 @@ public class RarParser implements Parser {
if (!header.isDirectory()) {
Metadata entrydata =
AbstractArchiveParser.handleEntryMetadata(
header.getFileName(), null, null,
header.getFullUnpackSize(), xhtml, context);
- // RAR4 times are DOS local time; junrar resolves them in
the JVM default zone
- AbstractArchiveParser.setLocalTime(entrydata,
FileSystem.CREATED, header.getCTime());
- AbstractArchiveParser.setLocalTime(entrydata,
FileSystem.MODIFIED, header.getMTime());
- AbstractArchiveParser.setLocalTime(entrydata,
FileSystem.ACCESSED, header.getATime());
+ if (rar.getFormat() == RarFormat.RAR50) {
+ // RAR5 stores instants, as Unix time or Windows
FILETIME
+ AbstractArchiveParser.setInstant(entrydata,
FileSystem.CREATED, header.getCTime());
+ AbstractArchiveParser.setInstant(entrydata,
FileSystem.MODIFIED, header.getMTime());
+ AbstractArchiveParser.setInstant(entrydata,
FileSystem.ACCESSED, header.getATime());
+ } else {
+ // RAR4 times are DOS local time; junrar resolves them
in the JVM default zone
+ AbstractArchiveParser.setLocalTime(entrydata,
FileSystem.CREATED, header.getCTime());
+ AbstractArchiveParser.setLocalTime(entrydata,
FileSystem.MODIFIED, header.getMTime());
+ AbstractArchiveParser.setLocalTime(entrydata,
FileSystem.ACCESSED, header.getATime());
+ }
EntryStream entryStream = new
EntryStream(rar.getInputStream(header));
try (TikaInputStream rarTis =
TikaInputStream.get(entryStream)) {
if (extractor.shouldParseEmbedded(entrydata, context))
{
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/test/java/org/apache/tika/parser/pkg/RarParserTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/test/java/org/apache/tika/parser/pkg/RarParserTest.java
index fc0e601e52..cae3f69a57 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/test/java/org/apache/tika/parser/pkg/RarParserTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/test/java/org/apache/tika/parser/pkg/RarParserTest.java
@@ -22,9 +22,13 @@ import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
+import java.io.ByteArrayOutputStream;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
+import java.util.TimeZone;
+import java.util.zip.CRC32;
import org.apache.commons.io.IOUtils;
import org.junit.jupiter.api.Test;
@@ -33,6 +37,8 @@ import org.xml.sax.ContentHandler;
import org.apache.tika.exception.EncryptedDocumentException;
import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.FileSystem;
+import org.apache.tika.metadata.HttpHeaders;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.parser.ParseContext;
@@ -119,6 +125,94 @@ public class RarParserTest extends AbstractPkgTest {
assertTrue(hasEmbeddedPath(metadataList, "/testXML.xml"));
}
+ @Test
+ public void testRar5(@TempDir Path tmp) throws Exception {
+ Path rar5 = Files.write(tmp.resolve("stored.rar"), storedRar5(-1,
+ "page-002.txt", "second page", "page-001.txt", "first page"));
+ List<Metadata> metadataList = getRecursiveMetadata(rar5, new
ParseContext(), false);
+ assertContains("version=5",
metadataList.get(0).get(HttpHeaders.CONTENT_TYPE));
+
assertNull(metadataList.get(0).get(TikaCoreProperties.CONTAINER_EXCEPTION));
+ assertEquals(3, metadataList.size());
+ assertTrue(hasEmbeddedPath(metadataList, "/page-001.txt"));
+ assertTrue(hasEmbeddedPath(metadataList, "/page-002.txt"));
+ assertContains("page-001.txt",
metadataList.get(0).get(TikaCoreProperties.TIKA_CONTENT));
+ }
+
+ /** RAR5 stores a Unix time, an instant: it does not move with the JVM's
time zone. */
+ @Test
+ public void testRar5TimesAreInstants(@TempDir Path tmp) throws Exception {
+ Path rar5 = Files.write(tmp.resolve("timed.rar"),
+ storedRar5(1_700_000_000L, "a.txt", "a"));
+ TimeZone zone = TimeZone.getDefault();
+ try {
+ for (String id : new String[]{"UTC", "America/New_York",
"Asia/Tokyo"}) {
+ TimeZone.setDefault(TimeZone.getTimeZone(id));
+ List<Metadata> metadataList = getRecursiveMetadata(rar5, new
ParseContext(), false);
+ assertEquals("2023-11-14T22:13:20Z",
+ metadataList.get(1).get(FileSystem.MODIFIED), id);
+ }
+ } finally {
+ TimeZone.setDefault(zone);
+ }
+ }
+
+ /**
+ * A RAR5 archive of stored (uncompressed) files, written after the RAR
5.0 archive format
+ * description (https://www.rarlab.com/technote.htm): signature, main
archive header, one
+ * file header with its data per file, end of archive header. A
modification time other
+ * than -1 is written as Unix time.
+ */
+ private static byte[] storedRar5(long mtime, String... namesAndContents) {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ out.writeBytes(new byte[]{'R', 'a', 'r', '!', 0x1a, 0x07, 0x01, 0x00});
+ out.writeBytes(rar5Block(concat(vint(1), vint(0), vint(0))));
+ for (int i = 0; i < namesAndContents.length; i += 2) {
+ byte[] name = namesAndContents[i].getBytes(StandardCharsets.UTF_8);
+ byte[] data = namesAndContents[i +
1].getBytes(StandardCharsets.UTF_8);
+ CRC32 crc = new CRC32();
+ crc.update(data);
+ byte[] time = mtime < 0 ? new byte[0] : littleEndian((int) mtime);
+ out.writeBytes(rar5Block(concat(vint(2), vint(0x0002),
vint(data.length),
+ vint(mtime < 0 ? 0x0004 : 0x0006), vint(data.length),
vint(0x20), time,
+ littleEndian((int) crc.getValue()), vint(0), vint(0),
vint(name.length),
+ name)));
+ out.writeBytes(data);
+ }
+ out.writeBytes(rar5Block(concat(vint(5), vint(0), vint(0))));
+ return out.toByteArray();
+ }
+
+ /** A header block: CRC32 of the size and the header, the size as a vint,
the header. */
+ private static byte[] rar5Block(byte[] header) {
+ byte[] sized = concat(vint(header.length), header);
+ CRC32 crc = new CRC32();
+ crc.update(sized);
+ return concat(littleEndian((int) crc.getValue()), sized);
+ }
+
+ private static byte[] vint(long value) {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ do {
+ int b = (int) (value & 0x7f);
+ value >>>= 7;
+ out.write(value != 0 ? b | 0x80 : b);
+ } while (value != 0);
+ return out.toByteArray();
+ }
+
+ private static byte[] littleEndian(int value) {
+ return new byte[]{(byte) value, (byte) (value >>> 8), (byte) (value
>>> 16),
+ (byte) (value >>> 24)};
+ }
+
+ private static byte[] concat(byte[]... parts) {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ for (byte[] part : parts) {
+ out.writeBytes(part);
+ }
+ return out.toByteArray();
+ }
+
private static boolean hasEmbeddedPath(List<Metadata> metadataList, String
path) {
return metadataList.stream()
.anyMatch(m ->
path.equals(m.get(TikaCoreProperties.EMBEDDED_RESOURCE_PATH)));
