This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 258c4cb6f3 TIKA-5065: improve file type precision for external tools
and libraries that process files (#3630)
258c4cb6f3 is described below
commit 258c4cb6f35f7e9b26aeebed04f9f1c9bf470c23
Author: Tim Allison <[email protected]>
AuthorDate: Tue Nov 17 09:02:57 2026 -0500
TIKA-5065: improve file type precision for external tools and libraries
that process files (#3630)
---
CHANGES.txt | 6 +
.../ROOT/pages/configuration/inference.adoc | 19 ++-
.../configuration/parsers/external-parser.adoc | 9 +
docs/modules/ROOT/pages/formats.adoc | 10 ++
.../org/apache/tika/embedder/ExternalEmbedder.java | 29 +++-
.../java/org/apache/tika/io/IsolatedInput.java | 175 +++++++++++++++++++
.../tika/parser/external/ExternalParser.java | 9 +-
.../tika/parser/external/ExternalParserConfig.java | 19 +++
.../apache/tika/parser/inference/MediaFormats.java | 83 +++++++++
.../tika/embedder/ExternalEmbedderInputTest.java | 55 ++++++
.../java/org/apache/tika/io/IsolatedInputTest.java | 139 +++++++++++++++
.../parser/external/ExternalParserInputTest.java | 84 +++++++++
.../org/apache/tika/parser/gdal/GDALParser.java | 189 +++++++++++----------
.../geoinfo/GeographicInformationParser.java | 6 +-
.../org/apache/tika/parser/grib/GribParser.java | 5 +-
.../java/org/apache/tika/parser/hdf/HDFParser.java | 9 +-
.../apache/tika/parser/isatab/ISArchiveParser.java | 30 +++-
.../apache/tika/parser/netcdf/NetCDFParser.java | 4 +-
.../apache/tika/parser/netcdf/NetcdfReaders.java | 115 +++++++++++++
.../apache/tika/parser/gdal/TestGDALParser.java | 47 ++++-
.../tika/parser/isatab/ISArchiveParserTest.java | 42 ++++-
.../tika/parser/netcdf/NetcdfReadersTest.java | 119 +++++++++++++
.../parser/scientific/integration/TestParsers.java | 22 ++-
.../java/org/apache/tika/inference/EmbedTask.java | 18 +-
.../org/apache/tika/inference/FfmpegSegmenter.java | 28 ++-
.../org/apache/tika/inference/MediaSegmenter.java | 7 +-
.../tika/inference/EmbedTaskMediaContractTest.java | 7 +-
.../apache/tika/inference/FfmpegSegmenterTest.java | 29 +++-
.../apache/tika/parser/ocr/ImagePreprocessor.java | 7 +-
.../org/apache/tika/parser/ocr/TesseractInput.java | 95 +++++++++++
.../apache/tika/parser/ocr/TesseractOCRParser.java | 72 +++++---
.../apache/tika/parser/ocr/TesseractInputTest.java | 83 +++++++++
.../tika/parser/ocr/TesseractOCRParserTest.java | 17 ++
.../java/org/apache/tika/parser/pkg/RarParser.java | 17 +-
.../org/apache/tika/parser/pkg/UnrarParser.java | 21 ++-
.../WireRestrictedParseContextTest.java | 26 +++
.../resources/configs/external-parser-ffmpeg.json | 4 +-
.../resources/configs/external-parser-multi.json | 4 +-
38 files changed, 1490 insertions(+), 170 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 4602fdbb64..aa3a3f85bf 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -2,6 +2,12 @@ Release 4.3.0 - ???
BREAKING CHANGES AND DEPRECATIONS
+ * External tools and libraries are told which formats to read. GDALParser
reads and
+ claims only the drivers in allowedFormats; media embedding reads only the
ffmpeg
+ formats in media-formats.allowedFormats; ExternalParser passes its input
as "input" plus the
+ detected type's extension (inputFileExtension overrides); ISArchiveParser
reads files
+ beside the study only with readSiblingFiles (TIKA-5065).
+
* tika-pipes-* and two tika-langdetect modules no longer pull in a log4j
backend;
embedding apps (and their PipesServer forks) must add an slf4j binding,
e.g.
log4j-slf4j2-impl (TIKA-5058).
diff --git a/docs/modules/ROOT/pages/configuration/inference.adoc
b/docs/modules/ROOT/pages/configuration/inference.adoc
index 26a79b5dab..00a06adb6f 100644
--- a/docs/modules/ROOT/pages/configuration/inference.adoc
+++ b/docs/modules/ROOT/pages/configuration/inference.adoc
@@ -361,7 +361,24 @@ Those are the defaults: 25 s windows, each starting 20 s
after the last, so any
5 s or less is whole in at least one segment, and no audio item reaches the 30
s cap the
hosted omni engines enforce (see <<jina-omni>>); `maxSegments` caps a
document, the rest of the
file is not embedded. A block with `overlap` outside `[0, seconds)`, or
`maxSegments` of 0
-or below -1, fails at config load and per request. The segmenter defines the
unit and every binding on the document sees
+or below -1, fails at config load and per request.
+
+ffmpeg reads a file only as one of the formats in `allowedFormats`, a list of
ffmpeg demuxer
+names in a parse-context block of its own:
+
+[source,json]
+----
+{ "parse-context": { "media-formats": { "allowedFormats": ["mov", "matroska",
"wav"] } } }
+----
+
+A file in any other format fails with an error that names the format. The
default is
+`mov`, `matroska`, `avi`, `asf`, `flv`, `mpegts`, `mpeg`, `mp3`, `wav`, `w64`,
`aiff`,
+`flac`, `ogg`, `aac`, `ac3`, `eac3`, `amr`, `caf`, `au`, `wv`, `ape`, `mxf`,
`rm`, `dv`,
+`m4v`, `mpegvideo`, `h264` and `hevc` (`mov` covers MP4, M4A and 3GP;
`matroska` covers
+WebM). Setting `allowedFormats` replaces that list, so restate the defaults
you want to
+keep. Only the operator's configuration can set `media-formats`, not a request.
+
+The segmenter defines the unit and every binding on the document sees
the same segments, so each segment's chunks share the segment's id: a chunk
carries a
`temporal` locator (`start_ms`, `end_ms` on the container's presentation
timeline), its
`producer` (the binding), its `modality` (`visual` or `audio`) and a
`correlator`, the
diff --git a/docs/modules/ROOT/pages/configuration/parsers/external-parser.adoc
b/docs/modules/ROOT/pages/configuration/parsers/external-parser.adoc
index f9eed90ea2..a2ace80794 100644
--- a/docs/modules/ROOT/pages/configuration/parsers/external-parser.adoc
+++ b/docs/modules/ROOT/pages/configuration/parsers/external-parser.adoc
@@ -69,6 +69,13 @@ decorators, not specific to `ExternalParser`.)
|`List<String>`
|The command and arguments to run. Use `$\{INPUT_FILE}` and `$\{OUTPUT_FILE}`
tokens for file paths.
+|`inputFileExtension`
+|`String`
+|Optional. The program is given the document as a file named `input` in a
temporary directory
+of its own. By default the file keeps the document's extension when that is an
extension of the media type Tika detected, and
+otherwise gets that type's usual extension (`input.wav` for `audio/vnd.wave`).
Set this to use
+one fixed extension instead, or `""` for none.
+
|`checkCommandLine`
|`List<String>`
|Optional. Command to verify the tool is installed (e.g., `["ffmpeg",
"-version"]`).
@@ -132,6 +139,8 @@ icon:github[]
https://github.com/apache/tika/blob/main/tika-serialization/src/te
Extracts audio/video metadata from `ffmpeg -i` output. FFmpeg writes metadata
to stderr, so this uses `stderrHandler`.
+The `-format_whitelist` option limits FFmpeg to the container formats listed.
+
[source,json]
----
include::example$external-parser-ffmpeg.json[]
diff --git a/docs/modules/ROOT/pages/formats.adoc
b/docs/modules/ROOT/pages/formats.adoc
index ac03265474..b498008cef 100644
--- a/docs/modules/ROOT/pages/formats.adoc
+++ b/docs/modules/ROOT/pages/formats.adoc
@@ -435,6 +435,16 @@
link:{api}/org/apache/tika/parser/isatab/ISArchiveParser.html[ISArchiveParser]
link:{api}/org/apache/tika/parser/netcdf/NetCDFParser.html[NetCDFParser]. These
pull in large native and scientific dependencies and are not in `tika-app`.
+`ISArchiveParser` parses the study file it is given. An ISA-Tab archive is a
directory, and
+the investigation file beside a study names assay files to read; the parser
reads those only
+when its `readSiblingFiles` setting is on (default: off).
+
+`GDALParser` runs the `gdalinfo` program. It lets `gdalinfo` try only the
drivers in its
+`allowedFormats` setting, and takes only the media types of those drivers. The
default is
+`GTiff`, `netCDF`, `GRIB`, `FITS`, `NITF` and `DTED`. Setting `allowedFormats`
(GDAL
+driver names, as `gdalinfo --formats` prints them) replaces that list. The
setting needs a
+GDAL whose `gdalinfo` has the `-if` option.
+
== Executable programs and libraries
link:{api}/org/apache/tika/parser/executable/ExecutableParser.html[ExecutableParser]
diff --git
a/tika-core/src/main/java/org/apache/tika/embedder/ExternalEmbedder.java
b/tika-core/src/main/java/org/apache/tika/embedder/ExternalEmbedder.java
index 4cdfbea850..bc7fa5e027 100644
--- a/tika-core/src/main/java/org/apache/tika/embedder/ExternalEmbedder.java
+++ b/tika-core/src/main/java/org/apache/tika/embedder/ExternalEmbedder.java
@@ -34,6 +34,7 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream;
import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.IsolatedInput;
import org.apache.tika.io.TemporaryResources;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
@@ -346,6 +347,7 @@ public class ExternalEmbedder implements Embedder {
TikaInputStream tikaInputStream = TikaInputStream.get(inputStream);
File tempOutputFile = null;
+ IsolatedInput isolated = null;
List<String> commandMetadataSegments = null;
if (hasMetadataCommandArguments) {
@@ -357,12 +359,21 @@ public class ExternalEmbedder implements Embedder {
List<String> cmd = new ArrayList<>();
for (String commandSegment : origCmd) {
if (commandSegment.contains(INPUT_FILE_TOKEN)) {
+ if (isolated == null) {
+ isolated = metadata == null ?
IsolatedInput.of(tikaInputStream.getPath())
+ : IsolatedInput.of(tikaInputStream.getPath(),
metadata);
+ }
commandSegment = commandSegment.replace(INPUT_FILE_TOKEN,
- tikaInputStream.getFile().toString());
+ isolated.getPath().toString());
inputToStdIn = false;
}
if (commandSegment.contains(OUTPUT_FILE_TOKEN)) {
- tempOutputFile = tmp.createTemporaryFile();
+ try {
+ tempOutputFile = tmp.createTemporaryFile();
+ } catch (IOException | RuntimeException e) {
+ IOUtils.closeQuietly(isolated);
+ throw e;
+ }
commandSegment = commandSegment
.replace(OUTPUT_FILE_TOKEN, tempOutputFile.toString());
outputFromStdOut = false;
@@ -401,10 +412,15 @@ public class ExternalEmbedder implements Embedder {
// Execute
Process process;
- if (cmd.toArray().length == 1) {
- process = Runtime.getRuntime().exec(cmd.toArray(String[]::new)[0]);
- } else {
- process = Runtime.getRuntime().exec(cmd.toArray(String[]::new));
+ try {
+ if (cmd.toArray().length == 1) {
+ process =
Runtime.getRuntime().exec(cmd.toArray(String[]::new)[0]);
+ } else {
+ process =
Runtime.getRuntime().exec(cmd.toArray(String[]::new));
+ }
+ } catch (IOException | RuntimeException e) {
+ IOUtils.closeQuietly(isolated);
+ throw e;
}
UnsynchronizedByteArrayOutputStream stdErrOutputStream =
UnsynchronizedByteArrayOutputStream.builder().get();
@@ -450,6 +466,7 @@ public class ExternalEmbedder implements Embedder {
// TemporaryResources.createTemporaryFile)
IOUtils.closeQuietly(tikaInputStream);
}
+ IOUtils.closeQuietly(isolated);
IOUtils.closeQuietly(outputStream);
IOUtils.closeQuietly(stdErrOutputStream);
if (process.exitValue() != 0) {
diff --git a/tika-core/src/main/java/org/apache/tika/io/IsolatedInput.java
b/tika-core/src/main/java/org/apache/tika/io/IsolatedInput.java
new file mode 100644
index 0000000000..3ec841cfd1
--- /dev/null
+++ b/tika-core/src/main/java/org/apache/tika/io/IsolatedInput.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.io;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.regex.Pattern;
+
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.tika.metadata.HttpHeaders;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.mime.MediaType;
+import org.apache.tika.mime.MimeType;
+import org.apache.tika.mime.MimeTypeException;
+import org.apache.tika.mime.MimeTypes;
+
+/**
+ * A document under a fixed name in a directory of its own, for handing to an
external
+ * program or a library that opens files by path.
+ * <p>
+ * The file is a hard link where the file system allows one; else a copy if
the document is
+ * small, a symbolic link if it is large, and a copy if neither link can be
made. Closing
+ * removes the directory and whatever the reader left in it; the source keeps
its content.
+ */
+public final class IsolatedInput implements Closeable {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(IsolatedInput.class);
+ // above this a copy costs more than it is worth; a symbolic link is used
instead
+ private static final long MAX_SMALL_COPY = 64L * 1024 * 1024;
+ private static final Pattern EXTENSION =
Pattern.compile("(\\.[A-Za-z0-9]{1,16})?");
+
+ private final Path dir;
+ private final Path path;
+
+ private IsolatedInput(Path dir, Path path) {
+ this.dir = dir;
+ this.path = path;
+ }
+
+ public static IsolatedInput of(Path source) throws IOException {
+ return of(source, "");
+ }
+
+ /**
+ * The file gets an extension of the media type Tika gave the document,
since some readers
+ * need one: the document's own extension if it is one of that type's,
else the type's
+ * usual one.
+ */
+ public static IsolatedInput of(Path source, Metadata metadata) throws
IOException {
+ return of(source, extensionFor(metadata));
+ }
+
+ static String extensionFor(Metadata metadata) {
+ MediaType type =
MediaType.parse(metadata.get(HttpHeaders.CONTENT_TYPE));
+ if (type == null) {
+ return "";
+ }
+ try {
+ MimeType registered = MimeTypes.getDefaultMimeTypes()
+ .getRegisteredMimeType(type.getBaseType().toString());
+ if (registered == null) {
+ return "";
+ }
+ String own = FilenameUtils.getSuffixFromPath(
+
String.valueOf(metadata.get(TikaCoreProperties.RESOURCE_NAME_KEY)));
+ for (String known : registered.getExtensions()) {
+ if (known.equalsIgnoreCase(own) &&
EXTENSION.matcher(own).matches()) {
+ return own;
+ }
+ }
+ String extension = registered.getExtension();
+ return EXTENSION.matcher(extension).matches() ? extension : "";
+ } catch (MimeTypeException e) {
+ return "";
+ }
+ }
+
+ /**
+ * @param extension appended to the fixed name, for a reader that needs one
+ */
+ public static IsolatedInput of(Path source, String extension) throws
IOException {
+ if (!EXTENSION.matcher(extension).matches()) {
+ throw new IllegalArgumentException("not a file extension: " +
extension);
+ }
+ return of(source, extension, Placement.values());
+ }
+
+ static IsolatedInput of(Path source, String extension, Placement...
placements)
+ throws IOException {
+ Path dir = Files.createTempDirectory("tika-input-");
+ Path target = dir.resolve("input" + extension);
+ try {
+ // the file itself, not a link to it
+ place(source.toRealPath(), target, placements);
+ } catch (IOException | RuntimeException e) {
+ FileUtils.deleteQuietly(dir.toFile());
+ throw e;
+ }
+ return new IsolatedInput(dir, target);
+ }
+
+ /** In the order tried. */
+ enum Placement {
+ HARD_LINK, SMALL_COPY, SYMBOLIC_LINK, COPY
+ }
+
+ private static void place(Path source, Path target, Placement[] placements)
+ throws IOException {
+ IOException last = null;
+ for (Placement placement : placements) {
+ try {
+ switch (placement) {
+ case HARD_LINK:
+ Files.createLink(target, source);
+ return;
+ case SMALL_COPY:
+ if (Files.size(source) > MAX_SMALL_COPY) {
+ continue;
+ }
+ Files.copy(source, target);
+ return;
+ case SYMBOLIC_LINK:
+ Files.createSymbolicLink(target, source);
+ return;
+ default:
+ Files.copy(source, target);
+ return;
+ }
+ } catch (IOException e) {
+ // another file system, a file this user may not link to, no
links here
+ last = e;
+ } catch (UnsupportedOperationException e) {
+ last = new IOException(e);
+ }
+ }
+ throw last != null ? last : new IOException("no way to place " +
source);
+ }
+
+ /**
+ * @return the document, as an absolute path
+ */
+ public Path getPath() {
+ return path;
+ }
+
+ @Override
+ public void close() {
+ try {
+ // does not follow links
+ FileUtils.deleteDirectory(dir.toFile());
+ } catch (IOException e) {
+ LOG.warn("couldn't delete {}", dir, e);
+ }
+ }
+}
diff --git
a/tika-core/src/main/java/org/apache/tika/parser/external/ExternalParser.java
b/tika-core/src/main/java/org/apache/tika/parser/external/ExternalParser.java
index 7c426006f7..dbb8e3787b 100644
---
a/tika-core/src/main/java/org/apache/tika/parser/external/ExternalParser.java
+++
b/tika-core/src/main/java/org/apache/tika/parser/external/ExternalParser.java
@@ -39,6 +39,7 @@ import org.apache.tika.config.ConfigDeserializer;
import org.apache.tika.config.JsonConfig;
import org.apache.tika.config.ParseTimeout;
import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.IsolatedInput;
import org.apache.tika.io.TemporaryResources;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.ExternalProcess;
@@ -160,9 +161,13 @@ public class ExternalParser implements Parser {
@Override
public void parse(TikaInputStream tis, ContentHandler handler, Metadata
metadata,
ParseContext context) throws IOException, SAXException,
TikaException {
- try (TemporaryResources tmp = new TemporaryResources()) {
+ // under a fixed name, in a directory of its own
+ try (TemporaryResources tmp = new TemporaryResources();
+ IsolatedInput input = config.getInputFileExtension() == null
+ ? IsolatedInput.of(tis.getPath(), metadata)
+ : IsolatedInput.of(tis.getPath(),
config.getInputFileExtension())) {
Path outFile = null;
- Path p = tis.getPath();
+ Path p = input.getPath();
List<String> thisCommandLine = new ArrayList<>();
Matcher inputMatcher = INPUT_TOKEN_MATCHER.matcher("");
Matcher outputMatcher = OUTPUT_TOKEN_MATCHER.matcher("");
diff --git
a/tika-core/src/main/java/org/apache/tika/parser/external/ExternalParserConfig.java
b/tika-core/src/main/java/org/apache/tika/parser/external/ExternalParserConfig.java
index b496422511..07a35b2372 100644
---
a/tika-core/src/main/java/org/apache/tika/parser/external/ExternalParserConfig.java
+++
b/tika-core/src/main/java/org/apache/tika/parser/external/ExternalParserConfig.java
@@ -45,6 +45,7 @@ public class ExternalParserConfig implements Serializable {
private long timeoutMillis = ExternalParser.DEFAULT_TIMEOUT_MILLIS;
private int maxStdErr = 10000;
private int maxStdOut = 10000;
+ private String inputFileExtension;
public ExternalParserConfig() {
}
@@ -160,4 +161,22 @@ public class ExternalParserConfig implements Serializable {
public void setMaxStdOut(int maxStdOut) {
this.maxStdOut = maxStdOut;
}
+
+ public String getInputFileExtension() {
+ return inputFileExtension;
+ }
+
+ /**
+ * The program is given the document as a file named {@code input} in a
directory of its
+ * own. By default the file keeps the document's extension if that is one
of the detected
+ * media type's, and otherwise gets that type's usual extension. Set this
to use one fixed
+ * extension instead, such as {@code ".wav"}, or the empty string for none.
+ */
+ public void setInputFileExtension(String inputFileExtension) {
+ if (inputFileExtension != null &&
!inputFileExtension.matches("(\\.[A-Za-z0-9]{1,16})?")) {
+ throw new IllegalArgumentException(
+ "inputFileExtension must be empty or a dot and letters or
digits");
+ }
+ this.inputFileExtension = inputFileExtension;
+ }
}
diff --git
a/tika-core/src/main/java/org/apache/tika/parser/inference/MediaFormats.java
b/tika-core/src/main/java/org/apache/tika/parser/inference/MediaFormats.java
new file mode 100644
index 0000000000..1e6bd3b30c
--- /dev/null
+++ b/tika-core/src/main/java/org/apache/tika/parser/inference/MediaFormats.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.inference;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.regex.Pattern;
+
+import org.apache.tika.annotation.TikaComponent;
+import org.apache.tika.config.Initializable;
+import org.apache.tika.config.OperatorOnly;
+import org.apache.tika.exception.TikaConfigException;
+import org.apache.tika.parser.ParseContext;
+
+/**
+ * The {@code "media-formats"} parse-context block: the ffmpeg demuxers audio
and video may be
+ * read with before a MEDIA binding sees them. A file in any other format is
refused.
+ * <p>
+ * Loaded from the {@code parse-context} section of the config and not
settable per request.
+ */
+@OperatorOnly
+@TikaComponent(name = "media-formats", spi = false)
+public class MediaFormats implements Serializable, Initializable {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final List<String> DEFAULT_ALLOWED_FORMATS = List.of("mov",
"matroska",
+ "avi", "asf", "flv", "mpegts", "mpeg", "mp3", "wav", "w64",
"aiff", "flac", "ogg",
+ "aac", "ac3", "eac3", "amr", "caf", "au", "wv", "ape", "mxf",
"rm", "dv", "m4v",
+ "mpegvideo", "h264", "hevc");
+
+ private static final Pattern FORMAT_NAME =
Pattern.compile("[A-Za-z0-9_]+");
+
+ private List<String> allowedFormats = DEFAULT_ALLOWED_FORMATS;
+
+ public List<String> getAllowedFormats() {
+ return allowedFormats;
+ }
+
+ /**
+ * This replaces {@link #DEFAULT_ALLOWED_FORMATS}, it does not add to it.
+ */
+ public void setAllowedFormats(List<String> allowedFormats) {
+ this.allowedFormats =
+ allowedFormats == null ? DEFAULT_ALLOWED_FORMATS :
List.copyOf(allowedFormats);
+ }
+
+ @Override
+ public void initialize() throws TikaConfigException {
+ if (allowedFormats.isEmpty()) {
+ throw new TikaConfigException(
+ "media-formats.allowedFormats must name at least one
format");
+ }
+ for (String format : allowedFormats) {
+ if (!FORMAT_NAME.matcher(format).matches()) {
+ throw new TikaConfigException(
+ "media-formats.allowedFormats: not a format name: " +
format);
+ }
+ }
+ }
+
+ /**
+ * @return the context's formats, or a new instance with the defaults
+ */
+ public static MediaFormats get(ParseContext context) {
+ MediaFormats formats = context == null ? null :
context.get(MediaFormats.class);
+ return formats != null ? formats : new MediaFormats();
+ }
+}
diff --git
a/tika-core/src/test/java/org/apache/tika/embedder/ExternalEmbedderInputTest.java
b/tika-core/src/test/java/org/apache/tika/embedder/ExternalEmbedderInputTest.java
new file mode 100644
index 0000000000..0150586478
--- /dev/null
+++
b/tika-core/src/test/java/org/apache/tika/embedder/ExternalEmbedderInputTest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.embedder;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+
+import java.io.ByteArrayOutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.DisabledOnOs;
+import org.junit.jupiter.api.condition.OS;
+import org.junit.jupiter.api.io.TempDir;
+
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.ParseContext;
+
+public class ExternalEmbedderInputTest {
+
+ @Test
+ @DisabledOnOs(OS.WINDOWS)
+ public void testInputIsGivenAsInputInItsOwnDirectory(@TempDir Path dir)
throws Exception {
+ Path doc = Files.write(dir.resolve("report.txt"),
"hello".getBytes(UTF_8));
+ ExternalEmbedder embedder = new ExternalEmbedder();
+ embedder.setCommand("echo", ExternalEmbedder.INPUT_FILE_TOKEN);
+ embedder.setMetadataCommandArguments(null);
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (TikaInputStream tis = TikaInputStream.get(doc)) {
+ embedder.embed(new Metadata(), tis, out, new ParseContext());
+ }
+ Path given = Paths.get(out.toString(UTF_8).trim());
+ assertEquals("input",
given.getFileName().toString().replaceFirst("\\..*", ""));
+ assertFalse(given.startsWith(dir));
+ assertFalse(Files.exists(given.getParent()));
+ }
+}
diff --git a/tika-core/src/test/java/org/apache/tika/io/IsolatedInputTest.java
b/tika-core/src/test/java/org/apache/tika/io/IsolatedInputTest.java
new file mode 100644
index 0000000000..9c0c2795a6
--- /dev/null
+++ b/tika-core/src/test/java/org/apache/tika/io/IsolatedInputTest.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.io;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+
+import org.apache.tika.metadata.HttpHeaders;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+
+public class IsolatedInputTest {
+
+ @Test
+ public void testFixedNameInItsOwnDirectory(@TempDir Path tmp) throws
Exception {
+ Path source = tmp.resolve("document.dat");
+ Files.write(source, "content".getBytes(UTF_8));
+ Files.write(tmp.resolve("other.dat"), "other".getBytes(UTF_8));
+ Path dir;
+ try (IsolatedInput input = IsolatedInput.of(source)) {
+ Path path = input.getPath();
+ dir = path.getParent();
+ assertTrue(path.isAbsolute());
+ assertEquals("input", path.getFileName().toString());
+ assertNotEquals(tmp, dir);
+ assertEquals(List.of("input"), names(dir));
+ assertArrayEquals("content".getBytes(UTF_8),
Files.readAllBytes(path));
+ // what a reader leaves behind goes with the directory
+ Files.write(dir.resolve("input.idx"), new byte[]{1});
+ }
+ assertFalse(Files.exists(dir));
+ assertArrayEquals("content".getBytes(UTF_8),
Files.readAllBytes(source));
+ assertEquals(List.of("document.dat", "other.dat"), names(tmp));
+ }
+
+ @Test
+ public void testExtensionOfTheMediaType(@TempDir Path tmp) throws
Exception {
+ Path source = tmp.resolve("document.dat");
+ Files.write(source, "content".getBytes(UTF_8));
+ Metadata metadata = new Metadata();
+ metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, "document.dat");
+ try (IsolatedInput input = IsolatedInput.of(source, metadata)) {
+ assertEquals("input", input.getPath().getFileName().toString());
+ }
+ metadata.set(HttpHeaders.CONTENT_TYPE, "application/pdf; charset=x");
+ try (IsolatedInput input = IsolatedInput.of(source, metadata)) {
+ assertEquals("input.pdf",
input.getPath().getFileName().toString());
+ }
+ // the document's own spelling is kept when it is one of the type's
extensions
+ metadata.set(HttpHeaders.CONTENT_TYPE, "image/jpeg");
+ metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, "photo.JPEG");
+ try (IsolatedInput input = IsolatedInput.of(source, metadata)) {
+ assertEquals("input.JPEG",
input.getPath().getFileName().toString());
+ }
+ // otherwise the type's usual one
+ metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, "photo.txt");
+ try (IsolatedInput input = IsolatedInput.of(source, metadata)) {
+ assertEquals("input.jpg",
input.getPath().getFileName().toString());
+ }
+ metadata.set(HttpHeaders.CONTENT_TYPE, "application/x-not-registered");
+ try (IsolatedInput input = IsolatedInput.of(source, metadata)) {
+ assertEquals("input", input.getPath().getFileName().toString());
+ }
+ }
+
+ @ParameterizedTest
+ @EnumSource(IsolatedInput.Placement.class)
+ public void
testEveryPlacementReadsTheSameAndLeavesTheSource(IsolatedInput.Placement
placement,
+ @TempDir Path
tmp)
+ throws Exception {
+ Path source = tmp.resolve("document.dat");
+ Files.write(source, "content".getBytes(UTF_8));
+ Path dir;
+ try (IsolatedInput input = IsolatedInput.of(source, ".dat",
placement)) {
+ dir = input.getPath().getParent();
+ assertEquals(List.of("input.dat"), names(dir));
+ assertArrayEquals("content".getBytes(UTF_8),
Files.readAllBytes(input.getPath()));
+ assertEquals(placement == IsolatedInput.Placement.SYMBOLIC_LINK,
+ Files.isSymbolicLink(input.getPath()));
+ } catch (IOException e) {
+ assumeTrue(false, placement + " is not available here: " + e);
+ return;
+ }
+ assertFalse(Files.exists(dir));
+ assertArrayEquals("content".getBytes(UTF_8),
Files.readAllBytes(source));
+ }
+
+ @Test
+ public void testSourceThatIsARelativeLink(@TempDir Path tmp) throws
Exception {
+ Files.write(tmp.resolve("document.dat"), "content".getBytes(UTF_8));
+ Path link = tmp.resolve("link.dat");
+ try {
+ Files.createSymbolicLink(link, Paths.get("document.dat"));
+ } catch (IOException | UnsupportedOperationException e) {
+ assumeTrue(false, "no symbolic links here: " + e);
+ }
+ try (IsolatedInput input = IsolatedInput.of(link)) {
+ assertArrayEquals("content".getBytes(UTF_8),
Files.readAllBytes(input.getPath()));
+ }
+ }
+
+ private static List<String> names(Path dir) throws Exception {
+ try (Stream<Path> s = Files.list(dir)) {
+ return s.map(p ->
p.getFileName().toString()).sorted().collect(Collectors.toList());
+ }
+ }
+}
diff --git
a/tika-core/src/test/java/org/apache/tika/parser/external/ExternalParserInputTest.java
b/tika-core/src/test/java/org/apache/tika/parser/external/ExternalParserInputTest.java
new file mode 100644
index 0000000000..98c729b55a
--- /dev/null
+++
b/tika-core/src/test/java/org/apache/tika/parser/external/ExternalParserInputTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.external;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.ExternalProcess;
+import org.apache.tika.metadata.HttpHeaders;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.sax.BodyContentHandler;
+import org.apache.tika.utils.ProcessUtils;
+
+public class ExternalParserInputTest {
+
+ /** The program is given the document under a fixed name in a directory of
its own. */
+ @Test
+ public void testInputFileNameAndDirectory(@TempDir Path tmp)
+ throws Exception {
+ assumeTrue(ProcessUtils.checkCommand(new String[]{"sh", "-c", "exit
0"}));
+ Path document = tmp.resolve("document.bin");
+ Files.write(document, "content".getBytes(UTF_8));
+ Files.write(tmp.resolve("other.bin"), "other".getBytes(UTF_8));
+
+ ExternalParserConfig config = new ExternalParserConfig();
+ config.setSupportedTypes(List.of("audio/vnd.wave"));
+ config.setCommandLine(List.of("sh", "-c",
+ "echo \"arg=$0\"; ls \"$(dirname \"$0\")\"; cat \"$0\"",
"${INPUT_FILE}"));
+ config.setReturnStdout(true);
+ Metadata metadata = new Metadata();
+ metadata.set(HttpHeaders.CONTENT_TYPE, "audio/vnd.wave");
+ try (TikaInputStream tis = TikaInputStream.get(document)) {
+ new ExternalParser(config).parse(tis, new BodyContentHandler(),
metadata,
+ new ParseContext());
+ }
+ String stdout = metadata.get(ExternalProcess.STD_OUT);
+ assertTrue(stdout.contains("content"), stdout);
+ assertTrue(stdout.contains("/input.wav\n"), stdout);
+ assertFalse(stdout.contains("document"), stdout);
+ assertFalse(stdout.contains("other"), stdout);
+ assertFalse(stdout.contains(tmp.toString()), stdout);
+ }
+
+ @Test
+ public void testInputFileExtension() {
+ ExternalParserConfig config = new ExternalParserConfig();
+ assertNull(config.getInputFileExtension());
+ config.setInputFileExtension(".wav");
+ assertEquals(".wav", config.getInputFileExtension());
+ config.setInputFileExtension("");
+ assertEquals("", config.getInputFileExtension());
+ assertThrows(IllegalArgumentException.class,
+ () -> config.setInputFileExtension("a.b"));
+ assertThrows(IllegalArgumentException.class, () ->
config.setInputFileExtension("wav"));
+ }
+}
diff --git
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/gdal/GDALParser.java
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/gdal/GDALParser.java
index 662e0bdfa5..bab2066d50 100644
---
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/gdal/GDALParser.java
+++
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/gdal/GDALParser.java
@@ -23,13 +23,14 @@ import static
org.apache.tika.parser.external.ExternalParser.INPUT_FILE_TOKEN;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
-import java.util.Arrays;
-import java.util.Collections;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Scanner;
import java.util.Set;
+import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -40,8 +41,9 @@ import org.xml.sax.SAXException;
import org.apache.tika.annotation.TikaComponent;
import org.apache.tika.config.ParseTimeout;
+import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.exception.TikaException;
-import org.apache.tika.io.TemporaryResources;
+import org.apache.tika.io.IsolatedInput;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.ExternalProcess;
import org.apache.tika.metadata.KeyPrefix;
@@ -97,67 +99,36 @@ public class GDALParser implements Parser {
private static final Property UPPER_RIGHT =
Property.externalText("gdal:upper-right");
private static final Property LOWER_RIGHT =
Property.externalText("gdal:lower-right");
- private static final Set<MediaType> SUPPORTED_TYPES =
Collections.unmodifiableSet(new HashSet<>(
- Arrays.asList(MediaType.application("x-netcdf"),
MediaType.application("vrt"),
- MediaType.image("geotiff"), MediaType.image("nitf"),
- MediaType.application("x-rpf-toc"),
MediaType.application("x-ecrg-toc"),
- MediaType.image("hfa"), MediaType.image("sar-ceos"),
MediaType.image("ceos"),
- MediaType.application("jaxa-pal-sar"),
MediaType.application("gff"),
- MediaType.application("elas"),
MediaType.application("aig"),
- MediaType.application("aaigrid"),
MediaType.application("grass-ascii-grid"),
- MediaType.application("sdts-raster"),
MediaType.application("dted"),
- MediaType.image("png"), MediaType.image("jpeg"),
MediaType.image("raster"),
- MediaType.application("jdem"), MediaType.image("gif"),
- MediaType.image("big-gif"), MediaType.image("envisat"),
MediaType.image("fits"),
- MediaType.application("fits"), MediaType.image("bsb"),
- MediaType.application("xpm"), MediaType.image("bmp"),
- MediaType.image("x-dimap"), MediaType.image("x-airsar"),
- MediaType.application("x-rs2"),
MediaType.application("x-pcidsk"),
- MediaType.application("pcisdk"),
MediaType.image("x-pcraster"),
- MediaType.image("ilwis"), MediaType.image("sgi"),
- MediaType.application("x-srtmhgt"),
MediaType.application("leveller"),
- MediaType.application("terragen"),
MediaType.application("x-gmt"),
- MediaType.application("x-isis3"),
MediaType.application("x-isis2"),
- MediaType.application("x-pds"),
MediaType.application("x-til"),
- MediaType.application("x-ers"),
MediaType.application("x-l1b"),
- MediaType.image("fit"), MediaType.application("x-grib"),
MediaType.image("jp2"),
- MediaType.application("x-rmf"),
MediaType.application("x-wcs"),
- MediaType.application("x-wms"),
MediaType.application("x-msgn"),
- MediaType.application("x-wms"),
MediaType.application("x-wms"),
- MediaType.application("x-rst"),
MediaType.application("x-ingr"),
- MediaType.application("x-gsag"),
MediaType.application("x-gsbg"),
- MediaType.application("x-gs7bg"),
MediaType.application("x-cosar"),
- MediaType.application("x-tsx"),
MediaType.application("x-coasp"),
- MediaType.application("x-r"),
MediaType.application("x-map"),
- MediaType.application("x-pnm"),
MediaType.application("x-doq1"),
- MediaType.application("x-doq2"),
MediaType.application("x-envi"),
- MediaType.application("x-envi-hdr"),
MediaType.application("x-generic-bin"),
- MediaType.application("x-p-aux"), MediaType.image("x-mff"),
- MediaType.image("x-mff2"), MediaType.image("x-fujibas"),
- MediaType.application("x-gsc"),
MediaType.application("x-fast"),
- MediaType.application("x-bt"),
MediaType.application("x-lan"),
- MediaType.application("x-cpg"), MediaType.image("ida"),
- MediaType.application("x-ndf"), MediaType.image("eir"),
- MediaType.application("x-dipex"),
MediaType.application("x-lcp"),
- MediaType.application("x-gtx"),
MediaType.application("x-los-las"),
- MediaType.application("x-ntv2"),
MediaType.application("x-ctable2"),
- MediaType.application("x-ace2"),
MediaType.application("x-snodas"),
- MediaType.application("x-kro"), MediaType.image("arg"),
- MediaType.application("x-rik"),
MediaType.application("x-usgs-dem"),
- MediaType.application("x-gxf"),
MediaType.application("x-dods"),
- MediaType.application("x-http"),
MediaType.application("x-bag"),
- MediaType.application("x-hdf"),
MediaType.image("x-hdf5-image"),
- MediaType.application("x-nwt-grd"),
MediaType.application("x-nwt-grc"),
- MediaType.image("adrg"), MediaType.image("x-srp"),
- MediaType.application("x-blx"),
MediaType.application("x-rasterlite"),
- MediaType.application("x-epsilon"),
MediaType.application("x-sdat"),
- MediaType.application("x-kml"),
MediaType.application("x-xyz"),
- MediaType.application("x-geo-pdf"),
MediaType.image("x-ozi"),
- MediaType.application("x-ctg"),
MediaType.application("x-e00-grid"),
- MediaType.application("x-zmap"),
MediaType.application("x-webp"),
- MediaType.application("x-ngs-geoid"),
MediaType.application("x-mbtiles"),
- MediaType.application("x-ppi"),
MediaType.application("x-cappi"))));
-
+ /**
+ * The GDAL drivers gdalinfo may open a file with, unless configured
otherwise.
+ */
+ public static final List<String> DEFAULT_ALLOWED_FORMATS =
+ List.of("GTiff", "netCDF", "GRIB", "FITS", "NITF", "DTED");
+
+ // the types this parser takes for each driver; a driver not listed here
adds none
+ private static final Map<String, Set<MediaType>> FORMAT_TYPES =
Map.ofEntries(
+ Map.entry("GTiff", Set.of(MediaType.image("geotiff"))),
+ Map.entry("netCDF", Set.of(MediaType.application("x-netcdf"))),
+ Map.entry("GRIB", Set.of(MediaType.application("x-grib"))),
+ Map.entry("FITS", Set.of(MediaType.image("fits"),
MediaType.application("fits"))),
+ Map.entry("NITF", Set.of(MediaType.image("nitf"))),
+ Map.entry("HFA", Set.of(MediaType.image("hfa"))),
+ Map.entry("DTED", Set.of(MediaType.application("dted"))),
+ Map.entry("HDF4", Set.of(MediaType.application("x-hdf"))),
+ Map.entry("HDF5", Set.of(MediaType.application("x-hdf"),
+ MediaType.image("x-hdf5-image"))),
+ Map.entry("ENVI", Set.of(MediaType.application("x-envi"),
+ MediaType.application("x-envi-hdr"))),
+ Map.entry("PNG", Set.of(MediaType.image("png"))),
+ Map.entry("JPEG", Set.of(MediaType.image("jpeg"))),
+ Map.entry("GIF", Set.of(MediaType.image("gif"))),
+ Map.entry("BMP", Set.of(MediaType.image("bmp"))),
+ Map.entry("WMS", Set.of(MediaType.application("x-wms"))),
+ Map.entry("WCS", Set.of(MediaType.application("x-wcs"))));
+
+ private static final Pattern FORMAT_NAME =
Pattern.compile("[A-Za-z0-9_]+");
+
+ private List<String> allowedFormats = DEFAULT_ALLOWED_FORMATS;
private String command;
@@ -179,26 +150,59 @@ public class GDALParser implements Parser {
this.command = command;
}
- public String processCommand(TikaInputStream tis) {
- String pCommand = this.command;
- try {
- if (this.command.contains(INPUT_FILE_TOKEN)) {
- pCommand = this.command.replace(INPUT_FILE_TOKEN,
tis.getFile().getPath());
+ public List<String> getAllowedFormats() {
+ return allowedFormats;
+ }
+
+ /**
+ * GDAL driver names (as <code>gdalinfo --formats</code> prints them) that
gdalinfo may
+ * open a file with. This replaces {@link #DEFAULT_ALLOWED_FORMATS}, it
does not add to
+ * it.
+ */
+ public void setAllowedFormats(List<String> allowedFormats) throws
TikaConfigException {
+ if (allowedFormats == null || allowedFormats.isEmpty()) {
+ throw new TikaConfigException("allowedFormats must name at least
one GDAL driver");
+ }
+ for (String format : allowedFormats) {
+ if (!FORMAT_NAME.matcher(format).matches()) {
+ throw new TikaConfigException("allowedFormats: not a GDAL
driver name: "
+ + format);
+ }
+ if (!FORMAT_TYPES.containsKey(format)) {
+ for (String known : FORMAT_TYPES.keySet()) {
+ if (known.equalsIgnoreCase(format)) {
+ throw new TikaConfigException(
+ "allowedFormats: " + format + " should be " +
known);
+ }
+ }
+ LOG.warn("allowedFormats: {} adds no media type; the names
with types are {}",
+ format, new TreeSet<>(FORMAT_TYPES.keySet()));
}
- } catch (Exception e) {
- LOG.warn("exception processing command", e);
}
-
- return pCommand;
+ this.allowedFormats = List.copyOf(allowedFormats);
}
// Split before substituting so whitespace in the input path stays inside
one argument.
- static String[] buildCommandLine(String command, String inputPath) {
- String[] args = command.split("\\s+", -1);
- for (int i = 0; i < args.length; i++) {
- args[i] = args[i].replace(INPUT_FILE_TOKEN, inputPath);
+ // Each -if names a driver gdalinfo may try; with any present it tries no
others.
+ static String[] buildCommandLine(String command, String inputPath,
+ List<String> allowedFormats) {
+ String[] split = command.split("\\s+", -1);
+ List<String> args = new ArrayList<>();
+ // before the input argument, or after the program if no argument is
the input alone
+ int formatsAt = 1;
+ for (int i = 0; i < split.length; i++) {
+ if (i > 0 && formatsAt == 1 && split[i].equals(INPUT_FILE_TOKEN)) {
+ formatsAt = i;
+ }
+ args.add(split[i].replace(INPUT_FILE_TOKEN, inputPath));
+ }
+ List<String> formats = new ArrayList<>();
+ for (String format : allowedFormats) {
+ formats.add("-if");
+ formats.add(format);
}
- return args;
+ args.addAll(formatsAt, formats);
+ return args.toArray(new String[0]);
}
// VRT and GTI open any file whose header merely contains their XML, so a
document of a
@@ -211,7 +215,11 @@ public class GDALParser implements Parser {
@Override
public Set<MediaType> getSupportedTypes(ParseContext context) {
- return SUPPORTED_TYPES;
+ Set<MediaType> types = new HashSet<>();
+ for (String format : allowedFormats) {
+ types.addAll(FORMAT_TYPES.getOrDefault(format, Set.of()));
+ }
+ return types;
}
@Override
@@ -222,16 +230,15 @@ public class GDALParser implements Parser {
return;
}
- // first set up and run GDAL
- // process the command
- TemporaryResources tmp = null;
-
- String[] runCommand = buildCommandLine(this.command,
tis.getFile().getPath());
-
- ProcessBuilder pb = new ProcessBuilder(runCommand);
- restrictEnvironment(pb.environment());
- FileProcessResult result = ProcessUtils.execute(pb, context,
- timeoutMillis, maxStdOut, maxStdErr);
+ FileProcessResult result;
+ // under a fixed name, in a directory of its own
+ try (IsolatedInput input = IsolatedInput.of(tis.getPath(), metadata)) {
+ String[] runCommand =
+ buildCommandLine(this.command, input.getPath().toString(),
allowedFormats);
+ ProcessBuilder pb = new ProcessBuilder(runCommand);
+ restrictEnvironment(pb.environment());
+ result = ProcessUtils.execute(pb, context, timeoutMillis,
maxStdOut, maxStdErr);
+ }
metadata.set(ExternalProcess.IS_TIMEOUT, result.isTimeout());
metadata.set(ExternalProcess.EXIT_VALUE, result.getExitValue());
@@ -243,6 +250,10 @@ public class GDALParser implements Parser {
metadata.set(ExternalProcess.STD_OUT, result.getStdout());
metadata.set(ExternalProcess.STD_ERR, result.getStderr());
+ if (result.getExitValue() != 0) {
+ throw new TikaException("GDALParser bad exit value " +
result.getExitValue()
+ + " err msg: " + result.getStderr());
+ }
String output = result.getStdout();
diff --git
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/geoinfo/GeographicInformationParser.java
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/geoinfo/GeographicInformationParser.java
index c0c858518f..abbf4b4957 100644
---
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/geoinfo/GeographicInformationParser.java
+++
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/geoinfo/GeographicInformationParser.java
@@ -61,6 +61,7 @@ import org.xml.sax.SAXException;
import org.apache.tika.annotation.TikaComponent;
import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.IsolatedInput;
import org.apache.tika.io.TemporaryResources;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.HttpHeaders;
@@ -112,8 +113,9 @@ public class GeographicInformationParser implements Parser {
XHTMLContentHandler xhtmlContentHandler = new
XHTMLContentHandler(contentHandler, metadata, parseContext);
TemporaryResources tmp = null;
- try (tis) {
- File file = tis.getFile();
+ // under a fixed name, in a directory of its own
+ try (tis; IsolatedInput input = IsolatedInput.of(tis.getPath(),
".xml")) {
+ File file = input.getPath().toFile();
try (DataStore dataStore = DataStores.open(file)) {
DefaultMetadata defaultMetadata = new
DefaultMetadata(dataStore.getMetadata());
extract(xhtmlContentHandler, metadata, defaultMetadata);
diff --git
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/grib/GribParser.java
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/grib/GribParser.java
index a2186cb036..970329dcb8 100644
---
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/grib/GribParser.java
+++
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/grib/GribParser.java
@@ -30,7 +30,6 @@ import ucar.nc2.Attribute;
import ucar.nc2.Dimension;
import ucar.nc2.NetcdfFile;
import ucar.nc2.Variable;
-import ucar.nc2.dataset.NetcdfDataset;
import org.apache.tika.annotation.TikaComponent;
import org.apache.tika.exception.TikaException;
@@ -44,6 +43,7 @@ import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.parser.climate.ClimateForecast;
+import org.apache.tika.parser.netcdf.NetcdfReaders;
import org.apache.tika.sax.XHTMLContentHandler;
@TikaComponent
@@ -75,7 +75,8 @@ public class GribParser implements Parser {
Path gribFile = Files.createTempFile(tmpDir, "tika-file",
".grib2");
Files.copy(tis, gribFile, StandardCopyOption.REPLACE_EXISTING);
- try (NetcdfFile ncFile =
NetcdfDataset.openFile(gribFile.toString(), null)) {
+ try (NetcdfFile ncFile = NetcdfReaders.open(gribFile,
NetcdfReaders.GRIB1,
+ NetcdfReaders.GRIB2)) {
// first parse out the set of global attributes
for (Attribute attr : ncFile.getGlobalAttributes()) {
diff --git
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/hdf/HDFParser.java
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/hdf/HDFParser.java
index 389a2188c7..513ab56221 100644
---
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/hdf/HDFParser.java
+++
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/hdf/HDFParser.java
@@ -36,11 +36,11 @@ import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.KeyPrefix;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.Property;
-import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.parser.netcdf.NetCDFParser;
+import org.apache.tika.parser.netcdf.NetcdfReaders;
import org.apache.tika.sax.XHTMLContentHandler;
/**
@@ -90,12 +90,9 @@ public class HDFParser implements Parser {
UnsynchronizedByteArrayOutputStream os =
UnsynchronizedByteArrayOutputStream.builder().get();
IOUtils.copy(tis, os);
- String name = metadata.get(TikaCoreProperties.RESOURCE_NAME_KEY);
- if (name == null) {
- name = "";
- }
try {
- NetcdfFile ncFile = NetcdfFile.openInMemory(name,
os.toByteArray());
+ NetcdfFile ncFile = NetcdfReaders.openInMemory(os.toByteArray(),
+ NetcdfReaders.HDF5, NetcdfReaders.HDF4);
unravelStringMet(ncFile, null, metadata);
} catch (IOException e) {
throw new TikaException("HDF parse error", e);
diff --git
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/isatab/ISArchiveParser.java
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/isatab/ISArchiveParser.java
index f7581f018e..9b0479cce2 100644
---
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/isatab/ISArchiveParser.java
+++
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/isatab/ISArchiveParser.java
@@ -45,6 +45,20 @@ public class ISArchiveParser implements Parser {
* Serial version UID
*/
private static final long serialVersionUID = 3640809327541300229L;
+
+ private boolean readSiblingFiles = false;
+
+ public boolean isReadSiblingFiles() {
+ return readSiblingFiles;
+ }
+
+ /**
+ * An ISA-Tab archive is a directory: the investigation file beside a
study names the
+ * assay files to read. Off by default, so only the study itself is parsed.
+ */
+ public void setReadSiblingFiles(boolean readSiblingFiles) {
+ this.readSiblingFiles = readSiblingFiles;
+ }
// Same isatab: KeyPrefix ISATabUtils writes these fields under -- must
stay in sync with
// ISATabUtils.addMetadata/mapStudyToMetadata or these reads silently
return nothing.
private static String studyAssayFileNameField =
ISATabUtils.ISATAB.key("Study Assay File Name");
@@ -68,15 +82,20 @@ public class ISArchiveParser implements Parser {
String resourceName =
metadata.get(TikaCoreProperties.RESOURCE_NAME_KEY);
String studyFileName = resourceName != null ? resourceName :
tis.getFile().getName();
- File locationFile = new File(location);
- String[] investigationList = locationFile.list((dir, name) ->
name.matches("i_.+\\.txt"));
XHTMLContentHandler xhtml = new XHTMLContentHandler(handler,
metadata, context);
xhtml.startDocument();
- parseInvestigation(location, studyFileName, investigationList,
xhtml, metadata, context);
+ if (readSiblingFiles) {
+ String[] investigationList =
+ new File(location).list((dir, name) ->
name.matches("i_.+\\.txt"));
+ parseInvestigation(location, studyFileName, investigationList,
xhtml, metadata,
+ context);
+ }
parseStudy(tis, xhtml, metadata, context);
- parseAssay(location, xhtml, metadata, context);
+ if (readSiblingFiles) {
+ parseAssay(location, xhtml, metadata, context);
+ }
xhtml.endDocument();
} finally {
@@ -109,7 +128,8 @@ public class ISArchiveParser implements Parser {
private void parseStudy(InputStream tis, XHTMLContentHandler xhtml,
Metadata metadata,
ParseContext context) throws IOException,
SAXException, TikaException {
- xhtml.element("h2", "STUDY " +
metadata.get(ISATabUtils.ISATAB.key("Study Identifier")));
+ String id = metadata.get(ISATabUtils.ISATAB.key("Study Identifier"));
+ xhtml.element("h2", id == null ? "STUDY" : "STUDY " + id);
ISATabUtils.parseStudy(tis, xhtml, metadata, context);
}
diff --git
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/netcdf/NetCDFParser.java
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/netcdf/NetCDFParser.java
index 3d88aae146..ecf4c9a9e2 100644
---
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/netcdf/NetCDFParser.java
+++
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/netcdf/NetCDFParser.java
@@ -32,6 +32,7 @@ import ucar.nc2.Variable;
import org.apache.tika.annotation.TikaComponent;
import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.IsolatedInput;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.KeyPrefix;
import org.apache.tika.metadata.Metadata;
@@ -88,7 +89,8 @@ public class NetCDFParser implements Parser {
public void parse(TikaInputStream tis, ContentHandler handler, Metadata
metadata,
ParseContext context) throws IOException, SAXException,
TikaException {
- try (NetcdfFile ncFile =
NetcdfFile.open(tis.getFile().getAbsolutePath())) {
+ try (IsolatedInput input = IsolatedInput.of(tis.getPath());
+ NetcdfFile ncFile = NetcdfReaders.open(input.getPath(),
NetcdfReaders.NETCDF3)) {
metadata.set(FILE_TYPE_DESCRIPTION,
ncFile.getFileTypeDescription());
// first parse out the set of global attributes
for (Attribute attr : ncFile.getGlobalAttributes()) {
diff --git
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/netcdf/NetcdfReaders.java
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/netcdf/NetcdfReaders.java
new file mode 100644
index 0000000000..93d47ac8b1
--- /dev/null
+++
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/netcdf/NetcdfReaders.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.netcdf;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+
+import org.apache.commons.io.IOUtils;
+import ucar.nc2.NetcdfFile;
+
+import org.apache.tika.exception.TikaException;
+
+/**
+ * Opens a file with the netcdf-java reader for its format, chosen from the
file's signature.
+ * A parser passes the readers for the formats it reads; any other file is
refused.
+ */
+public final class NetcdfReaders {
+
+ public static final String NETCDF3 = "ucar.nc2.iosp.netcdf3.N3raf";
+ public static final String HDF5 = "ucar.nc2.iosp.hdf5.H5iosp";
+ public static final String HDF4 = "ucar.nc2.iosp.hdf4.H4iosp";
+ public static final String GRIB1 = "ucar.nc2.grib.collection.Grib1Iosp";
+ public static final String GRIB2 = "ucar.nc2.grib.collection.Grib2Iosp";
+
+ private static final int HEAD_LENGTH = 8;
+ private static final int BUFFER_SIZE = 8092;
+
+ private NetcdfReaders() {
+ }
+
+ public static NetcdfFile open(Path file, String... allowed)
+ throws IOException, TikaException {
+ byte[] head = new byte[HEAD_LENGTH];
+ int n;
+ try (InputStream is = Files.newInputStream(file)) {
+ n = IOUtils.read(is, head);
+ }
+ String reader = allowedReader(head, n, allowed);
+ try {
+ return NetcdfFile.open(file.toAbsolutePath().toString(), reader,
BUFFER_SIZE, null,
+ null);
+ } catch (ReflectiveOperationException e) {
+ throw new TikaException("netcdf-java reader is not available: " +
reader, e);
+ } catch (RuntimeException e) {
+ throw new TikaException("netcdf-java could not read the file", e);
+ }
+ }
+
+ public static NetcdfFile openInMemory(byte[] data, String... allowed)
+ throws IOException, TikaException {
+ String reader = allowedReader(data, data.length, allowed);
+ try {
+ // a fixed name
+ return NetcdfFile.openInMemory("input", data, reader);
+ } catch (ReflectiveOperationException e) {
+ throw new TikaException("netcdf-java reader is not available: " +
reader, e);
+ } catch (RuntimeException e) {
+ throw new TikaException("netcdf-java could not read the file", e);
+ }
+ }
+
+ private static String allowedReader(byte[] head, int length, String[]
allowed)
+ throws TikaException {
+ String reader = forSignature(head, length);
+ if (reader == null || !Arrays.asList(allowed).contains(reader)) {
+ throw new TikaException("not a format this parser reads");
+ }
+ return reader;
+ }
+
+ /**
+ * @return the reader for the signature at the start of the file, or null
if none
+ */
+ static String forSignature(byte[] head, int length) {
+ if (length < HEAD_LENGTH) {
+ return null;
+ }
+ if (head[0] == 'C' && head[1] == 'D' && head[2] == 'F' && (head[3] ==
1 || head[3] == 2)) {
+ return NETCDF3;
+ }
+ if ((head[0] & 0xff) == 0x89 && head[1] == 'H' && head[2] == 'D' &&
head[3] == 'F'
+ && head[4] == '\r' && head[5] == '\n' && head[6] == 0x1a &&
head[7] == '\n') {
+ return HDF5;
+ }
+ if (head[0] == 0x0e && head[1] == 0x03 && head[2] == 0x13 && head[3]
== 0x01) {
+ return HDF4;
+ }
+ if (head[0] == 'G' && head[1] == 'R' && head[2] == 'I' && head[3] ==
'B') {
+ if (head[7] == 1) {
+ return GRIB1;
+ }
+ if (head[7] == 2) {
+ return GRIB2;
+ }
+ }
+ return null;
+ }
+}
diff --git
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/test/java/org/apache/tika/parser/gdal/TestGDALParser.java
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/test/java/org/apache/tika/parser/gdal/TestGDALParser.java
index ce060f9b3f..62131a240d 100644
---
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/test/java/org/apache/tika/parser/gdal/TestGDALParser.java
+++
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/test/java/org/apache/tika/parser/gdal/TestGDALParser.java
@@ -18,7 +18,9 @@ package org.apache.tika.parser.gdal;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
import static org.junit.jupiter.api.Assumptions.assumeTrue;
@@ -29,15 +31,19 @@ import java.net.ServerSocket;
import java.net.Socket;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
+import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import org.junit.jupiter.api.Test;
import org.apache.tika.TikaTest;
+import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.sax.BodyContentHandler;
import org.apache.tika.utils.ProcessUtils;
@@ -169,8 +175,21 @@ public class TestGDALParser extends TikaTest {
@Test
public void testInputPathWithWhitespaceIsOneArgument() {
String path = "/data/survey 2024/tile.tif --config GDAL_HTTP_PROXY
attacker:8080";
- assertArrayEquals(new String[]{"gdalinfo", path},
- GDALParser.buildCommandLine(new GDALParser().getCommand(),
path));
+ assertArrayEquals(new String[]{"gdalinfo", "-if", "GTiff", "-if",
"FITS", path},
+ GDALParser.buildCommandLine(new GDALParser().getCommand(),
path,
+ List.of("GTiff", "FITS")));
+ }
+
+ @Test
+ public void testFormatsPrecedeTheInputInAWrappedCommand() {
+ assertArrayEquals(new String[]{"docker", "run", "img", "gdalinfo",
"-if", "GTiff", "/in.tif",
+ "-json"},
+ GDALParser.buildCommandLine("docker run img gdalinfo
${INPUT_FILE} -json", "/in.tif",
+ List.of("GTiff")));
+ // not before an argument that only contains the input
+ assertArrayEquals(new String[]{"docker", "-if", "GTiff", "run", "-v",
"/in.tif:/in", "img"},
+ GDALParser.buildCommandLine("docker run -v ${INPUT_FILE}:/in
img", "/in.tif",
+ List.of("GTiff")));
}
@Test
@@ -220,4 +239,28 @@ public class TestGDALParser extends TikaTest {
assertEquals("JPEG VRT GTI", env.get("GDAL_SKIP"));
assertEquals("", env.get("CPL_VSIL_CURL_ALLOWED_FILENAME"));
}
+
+ @Test
+ public void testDefaultFormatsAreASmallSet() {
+ GDALParser parser = new GDALParser();
+ assertEquals(GDALParser.DEFAULT_ALLOWED_FORMATS,
parser.getAllowedFormats());
+ Set<MediaType> types = parser.getSupportedTypes(new ParseContext());
+ assertTrue(types.contains(MediaType.application("x-netcdf")));
+ assertTrue(types.contains(MediaType.image("fits")));
+ // not in the default formats
+ assertFalse(types.contains(MediaType.image("png")));
+ }
+
+ @Test
+ public void testAllowedFormatsReplacesTheDefault() throws Exception {
+ GDALParser parser = new GDALParser();
+ parser.setAllowedFormats(List.of("PNG"));
+ assertEquals(Set.of(MediaType.image("png")),
parser.getSupportedTypes(new ParseContext()));
+ assertThrows(TikaConfigException.class, () ->
parser.setAllowedFormats(List.of()));
+ assertThrows(TikaConfigException.class,
+ () -> parser.setAllowedFormats(List.of("GTiff HFA")));
+ assertThrows(TikaConfigException.class, () ->
parser.setAllowedFormats(List.of("gtiff")));
+ // a driver with no media type here is allowed
+ parser.setAllowedFormats(List.of("GTiff", "AAIGrid"));
+ }
}
diff --git
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/test/java/org/apache/tika/parser/isatab/ISArchiveParserTest.java
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/test/java/org/apache/tika/parser/isatab/ISArchiveParserTest.java
index 02308159e0..c530ee6ed1 100644
---
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/test/java/org/apache/tika/parser/isatab/ISArchiveParserTest.java
+++
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/test/java/org/apache/tika/parser/isatab/ISArchiveParserTest.java
@@ -19,6 +19,7 @@ package org.apache.tika.parser.isatab;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
@@ -45,7 +46,8 @@ public class ISArchiveParserTest extends TikaTest {
// the parser reads the investigation file next to the study, so open
the file itself
Path study = Paths.get(ISArchiveParserTest.class
.getResource("/test-documents/testISATab_BII-I-1/s_BII-S-1.txt").toURI());
- Parser parser = new ISArchiveParser();
+ ISArchiveParser parser = new ISArchiveParser();
+ parser.setReadSiblingFiles(true);
ContentHandler handler = new BodyContentHandler();
Metadata metadata = new Metadata();
@@ -94,7 +96,8 @@ public class ISArchiveParserTest extends TikaTest {
Path study = isaDir.resolve("s_test.txt");
Files.write(study, "\"Source
Name\"\n\"culture1\"\n".getBytes(StandardCharsets.UTF_8));
- Parser parser = new ISArchiveParser();
+ ISArchiveParser parser = new ISArchiveParser();
+ parser.setReadSiblingFiles(true);
ContentHandler handler = new BodyContentHandler();
Metadata metadata = new Metadata();
metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, "s_test.txt");
@@ -110,7 +113,8 @@ public class ISArchiveParserTest extends TikaTest {
@Test
public void testDirectoryNotCachedAcrossParses(@TempDir Path root) throws
Exception {
- Parser parser = new ISArchiveParser();
+ ISArchiveParser parser = new ISArchiveParser();
+ parser.setReadSiblingFiles(true);
assertEquals("FIRST", parseInvestigationId(parser, writeArchive(root,
"first", "FIRST")));
assertEquals("SECOND", parseInvestigationId(parser, writeArchive(root,
"second", "SECOND")));
}
@@ -137,4 +141,36 @@ public class ISArchiveParserTest extends TikaTest {
}
return metadata.get("isatab:Investigation Identifier");
}
+
+ @Test
+ public void testSiblingFilesAreNotReadByDefault(@TempDir Path root) throws
Exception {
+ Files.write(root.resolve("i_test.txt"),
+ ("STUDY\n"
+ + "Study File Name\t\"s_test.txt\"\n"
+ + "Study Assay File Name\t\"a_test.txt\"\n")
+ .getBytes(StandardCharsets.UTF_8));
+ Files.write(root.resolve("a_test.txt"),
+ "\"ASSAY_CONTENT\"\n".getBytes(StandardCharsets.UTF_8));
+ Path study = root.resolve("s_test.txt");
+ Files.write(study, "\"Source
Name\"\n\"culture1\"\n".getBytes(StandardCharsets.UTF_8));
+
+ ContentHandler handler = new BodyContentHandler();
+ Metadata metadata = new Metadata();
+ metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, "s_test.txt");
+ try (TikaInputStream tis = TikaInputStream.get(study)) {
+ new ISArchiveParser().parse(tis, handler, metadata, new
ParseContext());
+ }
+ assertTrue(handler.toString().contains("culture1"),
handler.toString());
+ assertFalse(handler.toString().contains("ASSAY_CONTENT"),
handler.toString());
+
+ ISArchiveParser optedIn = new ISArchiveParser();
+ optedIn.setReadSiblingFiles(true);
+ ContentHandler withSiblings = new BodyContentHandler();
+ Metadata optedInMetadata = new Metadata();
+ optedInMetadata.set(TikaCoreProperties.RESOURCE_NAME_KEY,
"s_test.txt");
+ try (TikaInputStream tis = TikaInputStream.get(study)) {
+ optedIn.parse(tis, withSiblings, optedInMetadata, new
ParseContext());
+ }
+ assertTrue(withSiblings.toString().contains("ASSAY_CONTENT"),
withSiblings.toString());
+ }
}
diff --git
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/test/java/org/apache/tika/parser/netcdf/NetcdfReadersTest.java
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/test/java/org/apache/tika/parser/netcdf/NetcdfReadersTest.java
new file mode 100644
index 0000000000..3a54f430e1
--- /dev/null
+++
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/test/java/org/apache/tika/parser/netcdf/NetcdfReadersTest.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.netcdf;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import org.apache.tika.TikaTest;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.Parser;
+import org.apache.tika.parser.grib.GribParser;
+import org.apache.tika.parser.hdf.HDFParser;
+import org.apache.tika.sax.BodyContentHandler;
+
+/**
+ * Each parser opens only its own formats.
+ */
+public class NetcdfReadersTest extends TikaTest {
+
+ private static final String NETCDF = "sresa1b_ncar_ccsm3_0_run1_200001.nc";
+ private static final String GRIB = "gdas1.forecmwf.2014062612.grib2";
+
+ @TempDir
+ Path tmp;
+
+ @Test
+ public void testNetCDFParserReadsNetCDF() throws Exception {
+ Path nc = copy(NETCDF, "a.nc");
+ Metadata metadata = new Metadata();
+ parse(new NetCDFParser(), TikaInputStream.get(nc), metadata);
+ assertTrue(metadata.names().length > 5, metadata.toString());
+ assertEquals(List.of("a.nc"), names(tmp));
+ }
+
+ @Test
+ public void testNetCDFParserRefusesAnotherFormat() throws Exception {
+ // a GRIB file
+ Path grib = copy(GRIB, "a.nc");
+ assertRefused(new NetCDFParser(), TikaInputStream.get(grib));
+ assertEquals(List.of("a.nc"), names(tmp));
+ }
+
+ @Test
+ public void testHDFParserRefusesAnotherFormat() throws Exception {
+ assertRefused(new HDFParser(), getResourceAsStream("/test-documents/"
+ NETCDF));
+ }
+
+ @Test
+ public void testGribParserRefusesAnotherFormat() throws Exception {
+ assertRefused(new GribParser(), TikaInputStream.get(copy(NETCDF,
"a.grib2")));
+ }
+
+ @Test
+ public void testReaderFollowsTheSignature() {
+ assertEquals(NetcdfReaders.NETCDF3,
+ NetcdfReaders.forSignature(new byte[]{'C', 'D', 'F', 1, 0, 0,
0, 0}, 8));
+ assertEquals(NetcdfReaders.GRIB2,
+ NetcdfReaders.forSignature(new byte[]{'G', 'R', 'I', 'B', 0,
0, 0, 2}, 8));
+ assertNull(NetcdfReaders.forSignature(new byte[]{'G', 'R', 'I', 'B',
0, 0, 0, 3}, 8));
+ assertNull(NetcdfReaders.forSignature(new byte[]{'a', 'b', 'c', 'd',
'e', 'f', 'g', 'h'}, 8));
+ assertNull(NetcdfReaders.forSignature(new byte[]{'C', 'D', 'F', 1, 0,
0, 0, 0}, 4));
+ }
+
+ private void assertRefused(Parser parser, TikaInputStream stream) {
+ TikaException e = assertThrows(TikaException.class,
+ () -> parse(parser, stream, new Metadata()));
+ assertEquals("not a format this parser reads", e.getMessage());
+ }
+
+ private void parse(Parser parser, TikaInputStream stream, Metadata
metadata)
+ throws Exception {
+ try (TikaInputStream tis = stream) {
+ parser.parse(tis, new BodyContentHandler(-1), metadata, new
ParseContext());
+ }
+ }
+
+ private Path copy(String resource, String name) throws Exception {
+ Path target = tmp.resolve(name);
+ try (InputStream is = getResourceAsStream("/test-documents/" +
resource)) {
+ Files.copy(is, target);
+ }
+ return target;
+ }
+
+ private static List<String> names(Path dir) throws Exception {
+ try (Stream<Path> s = Files.list(dir)) {
+ return s.map(p ->
p.getFileName().toString()).sorted().collect(Collectors.toList());
+ }
+ }
+}
diff --git
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-package/src/test/java/org/apache/tika/parser/scientific/integration/TestParsers.java
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-package/src/test/java/org/apache/tika/parser/scientific/integration/TestParsers.java
index 99e382c07d..776266c400 100644
---
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-package/src/test/java/org/apache/tika/parser/scientific/integration/TestParsers.java
+++
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-package/src/test/java/org/apache/tika/parser/scientific/integration/TestParsers.java
@@ -17,6 +17,7 @@
package org.apache.tika.parser.scientific.integration;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.BufferedReader;
@@ -24,14 +25,18 @@ import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Map;
+import java.util.Set;
import org.junit.jupiter.api.Test;
import org.apache.tika.exception.TikaException;
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.DefaultParser;
+import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
+import org.apache.tika.parser.gdal.GDALParser;
import org.apache.tika.parser.ocr.TesseractOCRParser;
/**
@@ -46,6 +51,8 @@ import org.apache.tika.parser.ocr.TesseractOCRParser;
*/
public class TestParsers {
+ private static final String GDAL = GDALParser.class.toString();
+
@Test
public void testDiffsFrom241() throws Exception {
@@ -55,6 +62,11 @@ public class TestParsers {
path241 = "/2.4.1-tesseract.txt";
}
+ Set<String> gdalTypes = new HashSet<>();
+ for (MediaType type : new GDALParser().getSupportedTypes(new
ParseContext())) {
+ gdalTypes.add(type.toString());
+ }
+
int checked = 0;
try (BufferedReader reader =
new BufferedReader(new InputStreamReader(
@@ -66,8 +78,14 @@ public class TestParsers {
String mediaType = data[0];
String parserClass = data[1];
- assertEquals(parserClass, currentDefault.get(mediaType),
- "for mediaType '" + mediaType + "'");
+ if (parserClass.equals(GDAL) &&
!gdalTypes.contains(mediaType)) {
+ // GDAL takes only the types of its allowedFormats; 2.4.1
gave it these too
+ assertNotEquals(GDAL, currentDefault.get(mediaType),
+ "for mediaType '" + mediaType + "'");
+ } else {
+ assertEquals(parserClass, currentDefault.get(mediaType),
+ "for mediaType '" + mediaType + "'");
+ }
checked++;
line = reader.readLine();
}
diff --git
a/tika-parsers/tika-parsers-ml/tika-inference/src/main/java/org/apache/tika/inference/EmbedTask.java
b/tika-parsers/tika-parsers-ml/tika-inference/src/main/java/org/apache/tika/inference/EmbedTask.java
index b9aba497ee..98aae6c3f6 100644
---
a/tika-parsers/tika-parsers-ml/tika-inference/src/main/java/org/apache/tika/inference/EmbedTask.java
+++
b/tika-parsers/tika-parsers-ml/tika-inference/src/main/java/org/apache/tika/inference/EmbedTask.java
@@ -45,6 +45,7 @@ import org.apache.tika.parser.inference.InferenceTask;
import org.apache.tika.parser.inference.InferenceUnit;
import org.apache.tika.parser.inference.InputKind;
import org.apache.tika.parser.inference.MediaConfig;
+import org.apache.tika.parser.inference.MediaFormats;
import org.apache.tika.parser.inference.Modality;
/**
@@ -253,6 +254,7 @@ public class EmbedTask implements InferenceTask {
EmbeddingEngine embedder, ParseContext
context)
throws IOException, TikaException {
MediaConfig config;
+ MediaFormats formats;
try {
config = ParseContextConfig.getConfig(context, "media",
MediaConfig.class,
new MediaConfig());
@@ -261,6 +263,12 @@ public class EmbedTask implements InferenceTask {
} catch (TikaConfigException e) {
throw new TikaException("invalid \"media\" in parse-context", e);
}
+ try {
+ formats = MediaFormats.get(context);
+ formats.initialize();
+ } catch (TikaConfigException e) {
+ throw new TikaException("invalid \"media-formats\" in
parse-context", e);
+ }
if (!segmenter.available()) {
for (InferenceUnit unit : units) {
unit.getDestination().add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING,
@@ -279,7 +287,7 @@ public class EmbedTask implements InferenceTask {
for (InferenceUnit unit : units) {
MediaSegmenter.Probe probe;
try {
- probe = segmenter.probe(unit.getPath(), context);
+ probe = segmenter.probe(unit.getPath(), formats, context);
} catch (MediaSegmenter.Timeout t) {
throw t;
} catch (TikaException e) {
@@ -299,7 +307,7 @@ public class EmbedTask implements InferenceTask {
cells.subList(start, Math.min(start + batchSize,
cells.size()));
try {
chunks.addAll(embedCells(binding, unit, batch,
modality, dir, embedder,
- context));
+ formats, context));
} catch (MediaSegmenter.Timeout t) {
throw t;
} catch (TikaException e) {
@@ -310,7 +318,7 @@ public class EmbedTask implements InferenceTask {
for (MediaSegmenter.Cell cell : batch) {
try {
chunks.addAll(embedCells(binding, unit,
List.of(cell), modality,
- dir, embedder, context));
+ dir, embedder, formats, context));
} catch (MediaSegmenter.Timeout t) {
throw t;
} catch (TikaException single) {
@@ -370,13 +378,13 @@ public class EmbedTask implements InferenceTask {
private static List<Chunk> embedCells(InferenceBinding binding,
InferenceUnit unit,
List<MediaSegmenter.Cell> cells,
Modality modality,
Path dir, EmbeddingEngine embedder,
- ParseContext context)
+ MediaFormats formats, ParseContext
context)
throws IOException, TikaException {
List<MediaInput> inputs = new ArrayList<>(cells.size());
List<Path> cuts = new ArrayList<>(cells.size());
String mime = MediaSegmenter.mimeType(modality);
for (MediaSegmenter.Cell cell : cells) {
- Path cut = segmenter.cut(unit.getPath(), cell, modality, dir,
context);
+ Path cut = segmenter.cut(unit.getPath(), cell, modality, dir,
formats, context);
if (binding.getMaxBytes() >= 0 && Files.size(cut) >
binding.getMaxBytes()) {
Files.deleteIfExists(cut);
throw new TikaException("segment " + cell.index() + " is over
maxBytes");
diff --git
a/tika-parsers/tika-parsers-ml/tika-inference/src/main/java/org/apache/tika/inference/FfmpegSegmenter.java
b/tika-parsers/tika-parsers-ml/tika-inference/src/main/java/org/apache/tika/inference/FfmpegSegmenter.java
index f60bbc2994..ecd7b06f0e 100644
---
a/tika-parsers/tika-parsers-ml/tika-inference/src/main/java/org/apache/tika/inference/FfmpegSegmenter.java
+++
b/tika-parsers/tika-parsers-ml/tika-inference/src/main/java/org/apache/tika/inference/FfmpegSegmenter.java
@@ -22,9 +22,12 @@ import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.apache.tika.exception.TikaException;
import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.inference.MediaFormats;
import org.apache.tika.parser.inference.Modality;
import org.apache.tika.utils.FileProcessResult;
import org.apache.tika.utils.ProcessUtils;
@@ -42,6 +45,9 @@ final class FfmpegSegmenter implements MediaSegmenter {
private static final int PROBE_BUFFER = 1_000_000;
private static final int CUT_BUFFER = 20_000;
+ private static final Pattern NOT_ALLOWED =
+ Pattern.compile("\\[([A-Za-z0-9_,]+) @ [^\\]]*\\] Format not on
whitelist");
+
private static volatile Boolean available;
/** Whether ffmpeg and ffprobe answer on the PATH; probed once per JVM,
outside any parse budget. */
@@ -66,8 +72,11 @@ final class FfmpegSegmenter implements MediaSegmenter {
}
@Override
- public Probe probe(Path media, ParseContext context) throws IOException,
TikaException {
- ProcessBuilder pb = new ProcessBuilder("ffprobe", "-v", "error",
"-show_entries",
+ public Probe probe(Path media, MediaFormats formats, ParseContext context)
+ throws IOException, TikaException {
+ ProcessBuilder pb = new ProcessBuilder("ffprobe", "-v", "error",
+ "-format_whitelist", String.join(",",
formats.getAllowedFormats()),
+ "-show_entries",
"format=duration:stream=codec_type:stream_disposition=attached_pic",
"-of", "default=nw=1", media.toAbsolutePath().toString());
FileProcessResult r = ProcessUtils.execute(pb, context,
PROBE_TIMEOUT_MS, PROBE_BUFFER,
@@ -77,6 +86,7 @@ final class FfmpegSegmenter implements MediaSegmenter {
+ r.getProcessTimeMillis() + " ms");
}
if (r.getExitValue() != 0) {
+ checkFormatAllowed(r.getStderr());
throw new TikaException("ffprobe failed: " + r.getStderr());
}
if (r.isStdoutTruncated()) {
@@ -116,7 +126,8 @@ final class FfmpegSegmenter implements MediaSegmenter {
}
@Override
- public Path cut(Path media, Cell cell, Modality modality, Path dir,
ParseContext context)
+ public Path cut(Path media, Cell cell, Modality modality, Path dir,
MediaFormats formats,
+ ParseContext context)
throws IOException, TikaException {
boolean audio = modality == Modality.AUDIO;
Path out = dir.resolve(cell.index() + (audio ? ".ogg" : ".mp4"));
@@ -125,6 +136,7 @@ final class FfmpegSegmenter implements MediaSegmenter {
}
List<String> cmd = new ArrayList<>(List.of("ffmpeg", "-hide_banner",
"-loglevel", "error",
"-y", "-ss", ms(cell.startMs()), "-t", ms(cell.endMs() -
cell.startMs()),
+ "-format_whitelist", String.join(",",
formats.getAllowedFormats()),
"-i", media.toAbsolutePath().toString()));
if (audio) {
cmd.addAll(List.of("-vn", "-ac", "1", "-c:a", "libopus", "-b:a",
"32k"));
@@ -144,12 +156,22 @@ final class FfmpegSegmenter implements MediaSegmenter {
+ cell.index() + " after " + r.getProcessTimeMillis() + "
ms");
}
if (r.getExitValue() != 0 || !Files.exists(out)) {
+ checkFormatAllowed(r.getStderr());
throw new TikaException("ffmpeg failed on segment " + cell.index()
+ ": "
+ r.getStderr());
}
return out;
}
+ // the name is one of ffmpeg's own demuxer names, not text from the file
+ private static void checkFormatAllowed(String stderr) throws TikaException
{
+ Matcher m = NOT_ALLOWED.matcher(stderr);
+ if (m.find()) {
+ throw new TikaException("ffmpeg format '" + m.group(1)
+ + "' is not in media-formats.allowedFormats");
+ }
+ }
+
private static String ms(long millis) {
return String.format(Locale.ROOT, "%d.%03d", millis / 1000, millis %
1000);
}
diff --git
a/tika-parsers/tika-parsers-ml/tika-inference/src/main/java/org/apache/tika/inference/MediaSegmenter.java
b/tika-parsers/tika-parsers-ml/tika-inference/src/main/java/org/apache/tika/inference/MediaSegmenter.java
index 21cbe9b3ef..15c1c5fa46 100644
---
a/tika-parsers/tika-parsers-ml/tika-inference/src/main/java/org/apache/tika/inference/MediaSegmenter.java
+++
b/tika-parsers/tika-parsers-ml/tika-inference/src/main/java/org/apache/tika/inference/MediaSegmenter.java
@@ -24,6 +24,7 @@ import java.util.List;
import org.apache.tika.exception.TikaException;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.inference.MediaConfig;
+import org.apache.tika.parser.inference.MediaFormats;
import org.apache.tika.parser.inference.Modality;
/**
@@ -51,13 +52,15 @@ interface MediaSegmenter {
/** Whether the tool answers; a MEDIA binding stands by without it. */
boolean available();
- Probe probe(Path media, ParseContext context) throws IOException,
TikaException;
+ Probe probe(Path media, MediaFormats formats, ParseContext context)
+ throws IOException, TikaException;
/**
* Writes one cell's audio or picture into {@code dir} and returns the
file. Idempotent
* per cell: an existing cut is returned, not redone, so a retried request
re-reads it.
*/
- Path cut(Path media, Cell cell, Modality modality, Path dir, ParseContext
context)
+ Path cut(Path media, Cell cell, Modality modality, Path dir, MediaFormats
formats,
+ ParseContext context)
throws IOException, TikaException;
/**
diff --git
a/tika-parsers/tika-parsers-ml/tika-inference/src/test/java/org/apache/tika/inference/EmbedTaskMediaContractTest.java
b/tika-parsers/tika-parsers-ml/tika-inference/src/test/java/org/apache/tika/inference/EmbedTaskMediaContractTest.java
index 2ba1d62d3f..ff78ab56b1 100644
---
a/tika-parsers/tika-parsers-ml/tika-inference/src/test/java/org/apache/tika/inference/EmbedTaskMediaContractTest.java
+++
b/tika-parsers/tika-parsers-ml/tika-inference/src/test/java/org/apache/tika/inference/EmbedTaskMediaContractTest.java
@@ -43,6 +43,7 @@ import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.inference.InferenceBinding;
import org.apache.tika.parser.inference.InferenceUnit;
import org.apache.tika.parser.inference.InputKind;
+import org.apache.tika.parser.inference.MediaFormats;
import org.apache.tika.parser.inference.Modality;
/**
@@ -71,7 +72,8 @@ public class EmbedTaskMediaContractTest {
}
@Override
- public Probe probe(Path media, ParseContext context) throws
TikaException {
+ public Probe probe(Path media, MediaFormats formats, ParseContext
context)
+ throws TikaException {
if (media.equals(failProbe)) {
throw new TikaException("ffprobe failed: not a media file");
}
@@ -79,7 +81,8 @@ public class EmbedTaskMediaContractTest {
}
@Override
- public Path cut(Path media, Cell cell, Modality modality, Path dir,
ParseContext context)
+ public Path cut(Path media, Cell cell, Modality modality, Path dir,
MediaFormats formats,
+ ParseContext context)
throws IOException, TikaException {
if (!cutDirs.contains(dir)) {
cutDirs.add(dir);
diff --git
a/tika-parsers/tika-parsers-ml/tika-inference/src/test/java/org/apache/tika/inference/FfmpegSegmenterTest.java
b/tika-parsers/tika-parsers-ml/tika-inference/src/test/java/org/apache/tika/inference/FfmpegSegmenterTest.java
index 4201b20ccd..1181810360 100644
---
a/tika-parsers/tika-parsers-ml/tika-inference/src/test/java/org/apache/tika/inference/FfmpegSegmenterTest.java
+++
b/tika-parsers/tika-parsers-ml/tika-inference/src/test/java/org/apache/tika/inference/FfmpegSegmenterTest.java
@@ -28,9 +28,11 @@ import java.util.List;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
+import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.exception.TikaException;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.inference.MediaConfig;
+import org.apache.tika.parser.inference.MediaFormats;
import org.apache.tika.parser.inference.Modality;
import org.apache.tika.utils.ProcessUtils;
@@ -112,9 +114,34 @@ public class FfmpegSegmenterTest {
"-y", "-i", plain.toString(), "-i", cover.toString(), "-map",
"0:a", "-map", "1:v",
"-c:a", "copy", "-c:v", "mjpeg", "-disposition:v",
"attached_pic",
tagged.toString()), 60_000, 10_000, 10_000).getExitValue());
- MediaSegmenter.Probe probe = segmenter.probe(tagged, new
ParseContext());
+ MediaSegmenter.Probe probe = segmenter.probe(tagged, new
MediaFormats(), new ParseContext());
assertTrue(probe.hasAudio());
assertFalse(probe.hasVideo());
assertTrue(probe.durationMs() >= 5000);
}
+
+ @Test
+ public void testAllowedFormatsReplacesTheDefault() throws Exception {
+ FfmpegSegmenter segmenter = new FfmpegSegmenter();
+ assumeTrue(segmenter.available(), "ffmpeg not on the PATH");
+ Path wav = tmp.resolve("a.wav");
+ assertEquals(0, ProcessUtils.execute(new ProcessBuilder("ffmpeg",
"-loglevel", "error",
+ "-y", "-f", "lavfi", "-i", "sine=frequency=440:duration=5",
+ wav.toString()), 60_000, 10_000, 10_000).getExitValue());
+ MediaFormats formats = new MediaFormats();
+ formats.setAllowedFormats(List.of("mp3"));
+ TikaException e = assertThrows(TikaException.class,
+ () -> segmenter.probe(wav, formats, new ParseContext()));
+ assertEquals("ffmpeg format 'wav' is not in
media-formats.allowedFormats",
+ e.getMessage());
+ }
+
+ @Test
+ public void testAllowedFormatsAreNames() {
+ MediaFormats formats = new MediaFormats();
+ formats.setAllowedFormats(List.of("wav,mp3"));
+ assertThrows(TikaConfigException.class, formats::initialize);
+ formats.setAllowedFormats(List.of());
+ assertThrows(TikaConfigException.class, formats::initialize);
+ }
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/ImagePreprocessor.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/ImagePreprocessor.java
index f1b9fbdb7e..f11659a7fa 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/ImagePreprocessor.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/ImagePreprocessor.java
@@ -76,8 +76,13 @@ class ImagePreprocessor implements Serializable {
final List<String> filter = Arrays.asList("-filter",
config.getFilter());
final List<String> resize = Arrays.asList("-resize",
config.getResize() + "%");
final List<String> rotate = Arrays.asList("-rotate",
Double.toString(-angle));
+ // names the format explicitly
+ String format = TesseractInput.format(sourceFile);
+ if (format == null) {
+ throw new IOException("not an image ImageMagick should be
given");
+ }
final List<String> sourceFileArg =
-
Collections.singletonList(sourceFile.toAbsolutePath().toString());
+ Collections.singletonList(format + ":" +
sourceFile.toAbsolutePath());
final List<String> targFileArg =
Collections.singletonList(targFile.toAbsolutePath().toString());
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/TesseractInput.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/TesseractInput.java
new file mode 100644
index 0000000000..89d03c8ad3
--- /dev/null
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/TesseractInput.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.ocr;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import org.apache.commons.io.IOUtils;
+
+/**
+ * The image formats handed to tesseract, recognized by the signature at the
start of the
+ * file.
+ */
+final class TesseractInput {
+
+ // the longest signature below
+ private static final int HEAD_LENGTH = 12;
+
+ private static final byte[][] SIGNATURES = {
+ {(byte) 0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n'},
+ {(byte) 0xff, (byte) 0xd8, (byte) 0xff},
+ {'I', 'I', 42, 0},
+ {'M', 'M', 0, 42},
+ {'I', 'I', 43, 0},
+ {'M', 'M', 0, 43},
+ {'B', 'M'},
+ {'G', 'I', 'F', '8', '7', 'a'},
+ {'G', 'I', 'F', '8', '9', 'a'},
+ {0, 0, 0, 0x0c, 'j', 'P', ' ', ' ', '\r', '\n', (byte) 0x87, '\n'}
+ };
+ // ImageMagick's name for each signature above
+ private static final String[] FORMATS = {"png", "jpeg", "tiff", "tiff",
"tiff", "tiff", "bmp", "gif", "gif",
+ "jp2"};
+
+ private TesseractInput() {
+ }
+
+ static boolean isAccepted(Path path) throws IOException {
+ return format(path) != null;
+ }
+
+ /**
+ * @return the image format the file's signature names, or null if it is
not accepted
+ */
+ static String format(Path path) throws IOException {
+ byte[] head = new byte[HEAD_LENGTH];
+ int n;
+ try (InputStream is = Files.newInputStream(path)) {
+ n = IOUtils.read(is, head);
+ }
+ return format(head, n);
+ }
+
+ static String format(byte[] head, int length) {
+ if (length < HEAD_LENGTH) {
+ return null;
+ }
+ for (int i = 0; i < SIGNATURES.length; i++) {
+ if (startsWith(head, SIGNATURES[i])) {
+ return FORMATS[i];
+ }
+ }
+ // PNM: P1 through P6 followed by whitespace
+ if (head[0] == 'P' && head[1] >= '1' && head[1] <= '6' &&
+ (head[2] == ' ' || head[2] == '\t' || head[2] == '\r' ||
head[2] == '\n')) {
+ return "pnm";
+ }
+ return null;
+ }
+
+ private static boolean startsWith(byte[] head, byte[] signature) {
+ for (int i = 0; i < signature.length; i++) {
+ if (head[i] != signature[i]) {
+ return false;
+ }
+ }
+ return true;
+ }
+}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/TesseractOCRParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/TesseractOCRParser.java
index 8639b88a8b..9c75db8f9f 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/TesseractOCRParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/TesseractOCRParser.java
@@ -48,6 +48,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.imageio.ImageIO;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -122,6 +123,7 @@ public class TesseractOCRParser extends
AbstractExternalProcessParser
private static final Logger LOG =
LoggerFactory.getLogger(TesseractOCRParser.class);
private static final Object[] LOCK = new Object[0];
+ private static final long STREAM_JOIN_MILLIS = 10000;
private static final long serialVersionUID = -8167538283213097265L;
private static final Set<MediaType> SUPPORTED_TYPES =
Collections.unmodifiableSet(new HashSet<>(
Arrays.asList(MediaType.image("png"), MediaType.image("jpeg"),
@@ -187,10 +189,10 @@ public class TesseractOCRParser extends
AbstractExternalProcessParser
Map<String, String> env = pb.environment();
if (!StringUtils.isBlank(defaultConfig.getTessdataPath())) {
- env.put(TESSDATA_PREFIX, defaultConfig.getTessdataPath());
+ env.put(TESSDATA_PREFIX,
absoluteDir(defaultConfig.getTessdataPath()));
} else if (!StringUtils.isBlank(defaultConfig.getTesseractPath())) {
//adding tessdata is required for at least >= 4.x
- env.put(TESSDATA_PREFIX, defaultConfig.getTesseractPath() +
"tessdata");
+ env.put(TESSDATA_PREFIX,
absoluteDir(defaultConfig.getTesseractPath()) + "tessdata");
}
}
@@ -350,12 +352,16 @@ public class TesseractOCRParser extends
AbstractExternalProcessParser
warnOnFirstParse();
validateLangString(config.getLanguage());
- File tmpTxtOutput = null;
+ String extension = config.getPageSegMode().equals("0") ? "osd" :
+ config.getOutputType().toString().toLowerCase(Locale.US);
+ // Tesseract appends the output type (.txt or .hocr or .osd) to output
file name
+ File tmpTxtOutput = new File(tmpOCROutputFile.getAbsolutePath() + "."
+ extension);
try {
Path input = tikaInputStream.getPath();
long size = tikaInputStream.getLength();
if (size >= config.getMinFileSizeToOcr() && size <=
config.getMaxFileSizeToOcr()) {
+ requireAccepted(input);
// Process image
if (config.isEnableImagePreprocessing() ||
config.isApplyRotation()) {
@@ -380,12 +386,6 @@ public class TesseractOCRParser extends
AbstractExternalProcessParser
doOCR(input.toFile(), tmpOCROutputFile, config,
parseContext);
}
- String extension = config.getPageSegMode().equals("0") ? "osd"
:
-
config.getOutputType().toString().toLowerCase(Locale.US);
- // Tesseract appends the output type (.txt or .hocr or .osd)
to output file name
- tmpTxtOutput = new File(tmpOCROutputFile.getAbsolutePath() +
- "." + extension);
-
if (tmpTxtOutput.exists()) {
try (InputStream is = new FileInputStream(tmpTxtOutput)) {
if (config.getPageSegMode().equals("0")) {
@@ -399,9 +399,7 @@ public class TesseractOCRParser extends
AbstractExternalProcessParser
}
}
} finally {
- if (tmpTxtOutput != null) {
- tmpTxtOutput.delete();
- }
+ tmpTxtOutput.delete();
}
}
@@ -458,10 +456,23 @@ public class TesseractOCRParser extends
AbstractExternalProcessParser
*/
private void doOCR(File input, File output, TesseractOCRConfig config,
ParseContext parseContext)
throws IOException, TikaException {
+ // rechecked here: with preprocessing this is ImageMagick's output
+ requireAccepted(input.toPath());
+ runTesseract(input, output, config, parseContext);
+ }
+
+ private static void requireAccepted(Path input) throws IOException,
TikaException {
+ if (!TesseractInput.isAccepted(input)) {
+ throw new TikaException("TesseractOCRParser input is not an image
tesseract accepts");
+ }
+ }
+
+ private void runTesseract(File input, File output, TesseractOCRConfig
config, ParseContext parseContext)
+ throws IOException, TikaException {
ArrayList<String> cmd = new ArrayList<>(
- Arrays.asList(defaultConfig.getTesseractPath() +
getTesseractProg(), input.getPath(),
- output.getPath(), "--psm", config.getPageSegMode()));
+ Arrays.asList(tesseractCommand(), input.getAbsolutePath(),
+ output.getAbsolutePath(), "--psm",
config.getPageSegMode()));
//if --psm == 0, don't add anything else to the command line
if (! "0".equals(config.getPageSegMode())) {
if (!StringUtils.isBlank(config.getLanguage())) {
@@ -481,6 +492,9 @@ public class TesseractOCRParser extends
AbstractExternalProcessParser
ProcessBuilder pb = new ProcessBuilder(cmd);
setEnv(pb);
+ // runs in a directory of its own
+ Path workingDir = Files.createTempDirectory("tika-ocr-");
+ pb.directory(workingDir.toFile());
Process process = null;
String id = null;
@@ -498,7 +512,21 @@ public class TesseractOCRParser extends
AbstractExternalProcessParser
if (id != null) {
release(id);
}
+ FileUtils.deleteQuietly(workingDir.toFile());
+ }
+ }
+
+ // absolute: the process does not run in this JVM's working directory
+ private String tesseractCommand() {
+ String dir = defaultConfig.getTesseractPath();
+ if (StringUtils.isBlank(dir)) {
+ return getTesseractProg();
}
+ return Paths.get(dir, getTesseractProg()).toAbsolutePath().toString();
+ }
+
+ private static String absoluteDir(String dir) {
+ return Paths.get(dir).toAbsolutePath() + File.separator;
}
private void runOCRProcess(Process process, ParseContext parseContext,
long requestedMillis,
@@ -527,13 +555,17 @@ public class TesseractOCRParser extends
AbstractExternalProcessParser
//this _should_ never be thrown
throw new TikaException("TesseractOCRParser timeout");
}
+ try {
+ outThread.join(STREAM_JOIN_MILLIS);
+ errThread.join(STREAM_JOIN_MILLIS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new TikaException("TesseractOCRParser interrupted", e);
+ }
+ if (outThread.isAlive() || errThread.isAlive()) {
+ throw new TikaException("TesseractOCRParser could not read the
process output");
+ }
if (exitValue > 0) {
- try {
- //make sure this thread is actually done
- errThread.join(1000);
- } catch (InterruptedException e) {
- //swallow
- }
throw new TikaException(
"TesseractOCRParser bad exit value " + exitValue + " err
msg: " +
errBuilder.toString());
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/test/java/org/apache/tika/parser/ocr/TesseractInputTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/test/java/org/apache/tika/parser/ocr/TesseractInputTest.java
new file mode 100644
index 0000000000..10cc674170
--- /dev/null
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/test/java/org/apache/tika/parser/ocr/TesseractInputTest.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.ocr;
+
+import static java.nio.charset.StandardCharsets.ISO_8859_1;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.nio.file.Paths;
+
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import org.apache.tika.TikaTest;
+
+public class TesseractInputTest extends TikaTest {
+
+ private static final String PAD = "\0\0\0\0\0\0\0\0\0\0\0\0";
+
+ @ParameterizedTest
+ @ValueSource(strings = {"testOCR_spacing.png", "testOCR.jpg",
"testTIFF_multipage.tif",
+ "testRotated+10.png"})
+ public void acceptsImages(String name) throws Exception {
+ assertTrue(TesseractInput.isAccepted(
+ Paths.get(getResourceAsUri("/test-documents/" + name))));
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {
+ "\u0089PNG\r\n\u001a\n\0\0\0\rIHDR",
+ "\u00ff\u00d8\u00ff\u00e0\0\u0010JFIF\0\u0001",
+ "II*\0\b\0\0\0\0\0\0\0",
+ "MM\0*\0\0\0\b\0\0\0\0",
+ "II+\0\b\0\0\0\0\0\0\0",
+ "MM\0+\0\b\0\0\0\0\0\0",
+ "BM\u0010\u0010\0\0\0\0\0\0\0\0",
+ "GIF89a\u0010\0\u0010\0\0\0",
+ "P6\n16 16\n255\n",
+ "P1 # ok\n1 1\n1\n\n",
+ "\0\0\0\fjP \r\n\u0087\n"})
+ public void acceptsSignatures(String head) {
+ assertTrue(accepted(head));
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {
+ "plain text\n" + PAD,
+ "short\n" + PAD,
+ "",
+ "BM\n",
+ "\u00ff\u00d8 plain text\n",
+ "GIF88a" + PAD,
+ "P6text text\n",
+ "P7\n" + PAD,
+ "RIFF\0\0\0\0WEBP",
+ "%PDF-1.7\n1 0 obj\n",
+ "%!PS-Adobe-3.0\n",
+ "spix" + PAD})
+ public void rejects(String head) {
+ assertFalse(accepted(head));
+ }
+
+ private static boolean accepted(String head) {
+ byte[] bytes = head.getBytes(ISO_8859_1);
+ byte[] buffer = new byte[Math.max(bytes.length, 12)];
+ System.arraycopy(bytes, 0, buffer, 0, bytes.length);
+ return TesseractInput.format(buffer, bytes.length) != null;
+ }
+}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/test/java/org/apache/tika/parser/ocr/TesseractOCRParserTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/test/java/org/apache/tika/parser/ocr/TesseractOCRParserTest.java
index 450183eac9..b89b186399 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/test/java/org/apache/tika/parser/ocr/TesseractOCRParserTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/test/java/org/apache/tika/parser/ocr/TesseractOCRParserTest.java
@@ -18,10 +18,12 @@ package org.apache.tika.parser.ocr;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
import static org.junit.jupiter.api.Assumptions.assumeTrue;
+import java.nio.charset.StandardCharsets;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -33,11 +35,13 @@ import org.apache.tika.config.TimeoutLimits;
import org.apache.tika.config.loader.TikaLoader;
import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.parser.inference.EngineRegistry;
+import org.apache.tika.sax.BodyContentHandler;
public class TesseractOCRParserTest extends TikaTest {
@@ -70,6 +74,19 @@ public class TesseractOCRParserTest extends TikaTest {
}
+ @Test
+ public void testNonImageIsRefused() throws Exception {
+ assumeTrue(canRun(), "can run OCR");
+ byte[] notAnImage = "this is not an
image".getBytes(StandardCharsets.UTF_8);
+ try (TikaInputStream tis = TikaInputStream.get(notAnImage)) {
+ TikaException e = assertThrows(TikaException.class,
+ () -> new TesseractOCRParser().parse(tis, new
BodyContentHandler(),
+ new Metadata(), new ParseContext()));
+ assertEquals("TesseractOCRParser input is not an image tesseract
accepts",
+ e.getMessage());
+ }
+ }
+
// no override: detection reaches Tesseract through the image parser
private Metadata getMetadata(MediaType mediaType) {
return new Metadata();
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/RarParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/RarParser.java
index c121232047..b57d1754cb 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/RarParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/RarParser.java
@@ -35,7 +35,7 @@ import org.apache.tika.exception.EncryptedDocumentException;
import org.apache.tika.exception.TikaException;
import org.apache.tika.extractor.EmbeddedDocumentExtractor;
import org.apache.tika.extractor.EmbeddedDocumentUtil;
-import org.apache.tika.io.TemporaryResources;
+import org.apache.tika.io.IsolatedInput;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.FileSystem;
import org.apache.tika.metadata.Metadata;
@@ -75,8 +75,10 @@ public class RarParser implements Parser {
EmbeddedDocumentExtractor extractor =
EmbeddedDocumentUtil.getEmbeddedDocumentExtractor(context);
Archive rar = null;
- try (TemporaryResources tmp = new TemporaryResources()) {
- rar = new Archive(tis.getFile(),
+ // under a fixed name, in a directory of its own
+ IsolatedInput input = IsolatedInput.of(tis.getPath(), ".rar");
+ try {
+ rar = new Archive(input.getPath().toFile(),
ArchiveOptions.builder().maxDictionarySize(MAX_DICTIONARY_SIZE).build());
if (rar.isEncrypted()) {
@@ -121,10 +123,13 @@ public class RarParser implements Parser {
} catch (RarException e) {
throw new TikaException("RarParser Exception", e);
} finally {
- if (rar != null) {
- rar.close();
+ try {
+ if (rar != null) {
+ rar.close();
+ }
+ } finally {
+ input.close();
}
-
}
xhtml.endDocument();
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/UnrarParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/UnrarParser.java
index d5ca7c2647..db2640dbe1 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/UnrarParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/UnrarParser.java
@@ -16,14 +16,17 @@
*/
package org.apache.tika.parser.pkg;
-import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.file.Files;
+import java.nio.file.LinkOption;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.Collections;
+import java.util.List;
import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
@@ -111,16 +114,20 @@ public class UnrarParser implements Parser {
xhtml.endDocument();
}
+ // regular files and directories only
private void processDirectory(Path baseDir, Path path,
XHTMLContentHandler xhtml,
EmbeddedDocumentExtractor extractor,
ParseContext context)
throws IOException, SAXException {
- for (File f : path.toFile().listFiles()) {
- if (f.isDirectory()) {
- processDirectory(baseDir, f.toPath(), xhtml, extractor,
- context);
- } else {
- processFile(baseDir, f.toPath(), xhtml, extractor, context);
+ List<Path> children;
+ try (Stream<Path> stream = Files.list(path)) {
+ children = stream.sorted().collect(Collectors.toList());
+ }
+ for (Path child : children) {
+ if (Files.isDirectory(child, LinkOption.NOFOLLOW_LINKS)) {
+ processDirectory(baseDir, child, xhtml, extractor, context);
+ } else if (Files.isRegularFile(child, LinkOption.NOFOLLOW_LINKS)) {
+ processFile(baseDir, child, xhtml, extractor, context);
}
}
}
diff --git
a/tika-serialization/src/test/java/org/apache/tika/serialization/WireRestrictedParseContextTest.java
b/tika-serialization/src/test/java/org/apache/tika/serialization/WireRestrictedParseContextTest.java
index ce092c655c..930bc7cd83 100644
---
a/tika-serialization/src/test/java/org/apache/tika/serialization/WireRestrictedParseContextTest.java
+++
b/tika-serialization/src/test/java/org/apache/tika/serialization/WireRestrictedParseContextTest.java
@@ -24,6 +24,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.HashSet;
+import java.util.List;
import java.util.Set;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -33,6 +34,7 @@ import org.junit.jupiter.api.Test;
import org.apache.tika.config.ExceptionReporting;
import org.apache.tika.config.loader.TikaObjectMapperFactory;
import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.inference.MediaFormats;
import org.apache.tika.serialization.serdes.ParseContextDeserializer;
/**
@@ -104,6 +106,30 @@ public class WireRestrictedParseContextTest {
"expected wire-blocked rejection, got: " + rootMessage(e));
}
+ @Test
+ public void restrictedRejectsMediaFormats() {
+ String json = "{\"media-formats\":{\"allowedFormats\":[\"wav\"]}}";
+ Exception e = assertThrows(Exception.class,
+ () -> restrictedMapper().readValue(json, ParseContext.class));
+ assertTrue(rootMessage(e).contains("may not be supplied via a request
parseContext"),
+ "expected wire-blocked rejection, got: " + rootMessage(e));
+ }
+
+ @Test
+ public void trustedAllowsMediaFormatsAndARequestMediaBlockKeepsThem()
throws Exception {
+ ObjectMapper mapper = TikaObjectMapperFactory.createMapper();
+ SimpleModule module = new SimpleModule();
+ module.addDeserializer(ParseContext.class, new
ParseContextDeserializer(false));
+ mapper.registerModule(module);
+ ParseContext merged = mapper.readValue(
+ "{\"media-formats\":{\"allowedFormats\":[\"wav\"]}}",
ParseContext.class);
+ // what a request may send: its own media block, laid over the
operator's context
+ merged.copyFrom(restrictedMapper().readValue(
+ "{\"media\":{\"maxSegments\":3}}", ParseContext.class));
+ ParseContextUtils.resolveAll(merged,
ParseContextUtils.class.getClassLoader());
+ assertEquals(List.of("wav"),
MediaFormats.get(merged).getAllowedFormats());
+ }
+
@Test
public void trustedAllowsExceptionReporting() throws Exception {
String json =
"{\"exception-reporting\":{\"level\":\"REDACTED\",\"maxLength\":100}}";
diff --git
a/tika-serialization/src/test/resources/configs/external-parser-ffmpeg.json
b/tika-serialization/src/test/resources/configs/external-parser-ffmpeg.json
index 3d9dd70bba..1429e69611 100644
--- a/tika-serialization/src/test/resources/configs/external-parser-ffmpeg.json
+++ b/tika-serialization/src/test/resources/configs/external-parser-ffmpeg.json
@@ -7,7 +7,9 @@
"video/mpeg",
"video/x-msvideo"
],
- "commandLine": ["ffmpeg", "-i", "${INPUT_FILE}"],
+ "commandLine": ["ffmpeg", "-format_whitelist",
+
"mov,matroska,avi,asf,flv,mpegts,mpeg,mp3,wav,w64,aiff,flac,ogg,aac,ac3,eac3,amr,caf,au,wv,ape,mxf,rm,dv,m4v,mpegvideo,h264,hevc",
+ "-i", "${INPUT_FILE}"],
"checkCommandLine": ["ffmpeg", "-version"],
"checkErrorCodes": [126, 127],
"contentSource": "none",
diff --git
a/tika-serialization/src/test/resources/configs/external-parser-multi.json
b/tika-serialization/src/test/resources/configs/external-parser-multi.json
index cce81a55fe..31528f209f 100644
--- a/tika-serialization/src/test/resources/configs/external-parser-multi.json
+++ b/tika-serialization/src/test/resources/configs/external-parser-multi.json
@@ -7,7 +7,9 @@
"video/mpeg",
"video/x-msvideo"
],
- "commandLine": ["ffmpeg", "-i", "${INPUT_FILE}"],
+ "commandLine": ["ffmpeg", "-format_whitelist",
+
"mov,matroska,avi,asf,flv,mpegts,mpeg,mp3,wav,w64,aiff,flac,ogg,aac,ac3,eac3,amr,caf,au,wv,ape,mxf,rm,dv,m4v,mpegvideo,h264,hevc",
+ "-i", "${INPUT_FILE}"],
"checkCommandLine": ["ffmpeg", "-version"],
"checkErrorCodes": [126, 127],
"returnStderr": true,