This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 159edb2a1a TIKA-4881 (#3178)
159edb2a1a is described below
commit 159edb2a1ae1f0b8b2e50242a42ee7e715f9ed06
Author: Tim Allison <[email protected]>
AuthorDate: Tue Sep 15 15:55:57 2026 -0400
TIKA-4881 (#3178)
---
CHANGES.txt | 4 +
.../migration-to-4x/migrating-tika-server-4x.adoc | 4 +-
docs/modules/ROOT/pages/pipes/unpack-config.adoc | 128 +++++++++-
docs/modules/ROOT/pages/using-tika/cli/index.adoc | 14 +-
.../ROOT/pages/using-tika/server/index.adoc | 23 +-
.../src/main/java/org/apache/tika/cli/TikaCLI.java | 7 +-
.../test/java/org/apache/tika/cli/TikaCLITest.java | 46 ++++
.../configs/tika-config-unpack-original.json | 7 +
.../org/apache/tika/async/cli/TikaAsyncCLI.java | 37 ++-
.../tika/pipes/core/extractor/UnpackConfig.java | 90 +++++--
.../apache/tika/pipes/core/server/PipesWorker.java | 6 +-
.../extractor/UnpackConfigDeprecatedFlagsTest.java | 94 ++++++++
.../core/serialization/JsonFetchEmitTupleTest.java | 9 +-
.../server/core/resource/PipesParsingHelper.java | 6 +-
.../core/resource/RecursiveMetadataResource.java | 36 ++-
.../tika/server/core/resource/TikaResource.java | 44 +++-
.../server/core/resource/UnpackerResource.java | 120 +++++++--
.../core/RecursiveMetadataConfigHandlerTest.java | 174 ++++++++++++++
.../server/core/TikaResourceConfigFormatTest.java | 116 +++++++++
.../core/resource/RequestContextIsolationTest.java | 15 ++
.../standard/RecursiveMetadataResourceTest.java | 1 -
.../server/standard/UnpackFrictionlessTest.java | 194 +++++++++++++++
.../standard/UnpackerResourceHandlerTest.java | 267 +++++++++++++++++++++
.../tika/server/standard/UnpackerResourceTest.java | 8 +-
24 files changed, 1359 insertions(+), 91 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 1c7aeca008..c2a3962d0f 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,9 @@
Release 4.1.0 - unreleased
+ * unpack-config gains includeMetadata (TIKA-4881).
+
+ * tika-server: /rmeta gains config/{handlerType} (TIKA-4881).
+
* An inference engine named under "parsers" (an OCR engine, a VLM parser,
openai-image-embedding-parser) is deprecated: since 4.1.0 startup logs
one WARN per such entry saying what it does today, and in 4.2.0 the
diff --git
a/docs/modules/ROOT/pages/migration-to-4x/migrating-tika-server-4x.adoc
b/docs/modules/ROOT/pages/migration-to-4x/migrating-tika-server-4x.adoc
index a0d86a47ba..6ef28fc200 100644
--- a/docs/modules/ROOT/pages/migration-to-4x/migrating-tika-server-4x.adoc
+++ b/docs/modules/ROOT/pages/migration-to-4x/migrating-tika-server-4x.adoc
@@ -412,8 +412,8 @@ The `X-Tika-OCR*` and `X-Tika-PDF*` header families were
removed earlier in the
**Migration:** move these settings into your tika-config. For per-request
values,
POST to a `/config` endpoint (`/tika/config`, `/tika/config/json`,
`/rmeta/config`,
-`/meta/config`) with the JSON above as the multipart `config` part; this
requires
-`allowPerRequestConfig=true`.
+`/rmeta/config/{handlerType}`, `/meta/config`) with the JSON above as the
multipart `config`
+part; this requires `allowPerRequestConfig=true`.
[IMPORTANT]
====
diff --git a/docs/modules/ROOT/pages/pipes/unpack-config.adoc
b/docs/modules/ROOT/pages/pipes/unpack-config.adoc
index 4884b2689e..af8560ae29 100644
--- a/docs/modules/ROOT/pages/pipes/unpack-config.adoc
+++ b/docs/modules/ROOT/pages/pipes/unpack-config.adoc
@@ -97,10 +97,15 @@ immediately capped.
|`false`
|Collect all embedded files into a single ZIP archive, emitted at the
container's emit key plus `-embedded.zip`.
+|`includeMetadata`
+|boolean
+|unset: on for `FRICTIONLESS`, off for `REGULAR`
+|Metadata for every extracted file, in either format: one `metadata.json`
under Frictionless (a data package without the parse is half a package, so it
is on unless you set `false`), a `.metadata.json` sidecar per file under
REGULAR zip output (opt-in; tika-server's `/unpack/all` sets it). REGULAR loose
output (no zip) writes none.
+
|`includeMetadataInZip`
|boolean
-|`false`
-|Include `.metadata.json` files for each embedded document in the ZIP.
+|--
+|Deprecated alias for `includeMetadata` (removal in 5.0). Still accepted so
4.0.0 configs load.
|`zeroPadName`
|int
@@ -139,8 +144,8 @@ immediately capped.
|`includeFullMetadata`
|boolean
-|`false`
-|Include a `metadata.json` file with full RMETA-style metadata for all
extracted files. Frictionless output only.
+|--
+|Deprecated alias for `includeMetadata` (removal in 5.0). Still accepted so
4.0.0 configs load.
|===
== Examples
@@ -158,7 +163,7 @@ Collect all embedded files into a ZIP with metadata:
"parse-context": {
"unpack-config": {
"zipEmbeddedFiles": true,
- "includeMetadataInZip": true,
+ "includeMetadata": true,
"includeOriginal": true
}
}
@@ -233,7 +238,7 @@ Set `outputFormat` to `FRICTIONLESS` in your
`unpack-config`:
"parse-context": {
"unpack-config": {
"outputFormat": "FRICTIONLESS",
- "includeFullMetadata": true
+ "includeMetadata": true
}
}
}
@@ -247,13 +252,122 @@ When using Frictionless output format, the ZIP archive
contains:
----
output.zip
├── datapackage.json # Manifest with file list, SHA256 hashes, mimetypes
-├── metadata.json # Full RMETA metadata (if includeFullMetadata=true)
+├── metadata.json # Full RMETA metadata (unless includeMetadata=false)
└── unpacked/
├── 00000001.pdf
├── 00000002.png
└── ...
----
+The `tk:content` field inside `metadata.json` (and inside the per-file
`.metadata.json`
+sidecars `includeMetadata` adds to REGULAR zip output) is Markdown by default.
On tika-server
+change it with `/unpack/all/{handlerType}` or a `content-handler-factory` in
the `config`
+part; in a config, the factory alone. Each entry records the handler it was
written with in
+`tk:content-handler-type`:
+
+[source,json]
+----
+{
+ "unpack-config": { "outputFormat": "FRICTIONLESS" },
+ "basic-content-handler-factory": { "type": "XML" }
+}
+----
+
+To get metadata with no extracted text at all -- one sidecar per file, but no
`tk:content` --
+select the `IGNORE` handler. On the test corpus below this drops the sidecars
from 20,257 to
+11,132 bytes and is the way to avoid shipping every embedded document's text
alongside its
+bytes:
+
+[source,json]
+----
+{
+ "unpack-config": { "includeMetadata": true },
+ "basic-content-handler-factory": { "type": "IGNORE" }
+}
+----
+
+On tika-server the handler is also a path segment on
`/unpack/all/{handlerType}`, which
+needs no `config` part and so no `allowPerRequestConfig`. Plain `/unpack`
carries no
+metadata for a handler to render, so it takes no segment.
+
+[source,bash]
+----
+curl -T container.docx http://localhost:9998/unpack/all/ignore # files +
metadata, no text
+----
+
+Naming the handler in both the path and a `config` part is a `400`, as on
`/tika` and
+`/rmeta`.
+
+== What each surface produces
+
+The five knobs interact differently per surface. One default changes from
4.0.0: a
+Frictionless package carries `metadata.json` unless `includeMetadata` is
`false` (4.0.0 left
+it out unless `includeFullMetadata` was `true`). REGULAR output is as it was.
+
+[cols="1,2,2,2"]
+|===
+| |tika-server `/unpack` |tika-app `-z` / `-Z` |pipes JSON config
+
+|format
+|REGULAR; set `outputFormat` in the server config for Frictionless
+|REGULAR; `--unpack-format=FRICTIONLESS`
+|`outputFormat`
+
+|packaging
+|always one zip (`ZIPPED` is pinned; an HTTP response is one body)
+|loose files; `DIRECTORY` when Frictionless is asked for without
`--unpack-mode`
+|`outputMode` governs Frictionless, `zipEmbeddedFiles` governs REGULAR
+
+|depth
+|`embedded-limits` in config
+|`-z` = direct attachments, `-Z` = recursive
+|`embedded-limits`
+
+|metadata
+|Frictionless: by default; REGULAR: `/unpack/all`
+|Frictionless: by default; REGULAR zip: `--unpack-include-metadata`
+|`includeMetadata` (unset: on for Frictionless, off for REGULAR)
+
+|source document's bytes
+|`/unpack/all`
+|only via `-c` (`unpack-config.includeOriginal`)
+|`includeOriginal`
+
+|handler for `tk:content`
+|`/unpack/all/{handlerType}` or the config part
+|`--handler`
+|`content-handler-factory`
+|===
+
+`/unpack/all` means the source document's bytes plus metadata in both formats.
Under REGULAR
+that is where the per-file sidecars come from; under Frictionless
`metadata.json` is already
+there, so `/all` adds only the source document. `outputMode: DIRECTORY` is
refused on
+`/unpack`: an HTTP response is one zip, so a request asking for it gets a
`400`, and a server
+whose own `unpack-config` says so fails at startup if the `unpack` endpoint is
enabled -- use
+`/pipes` or `/async` for per-file emission.
+
+Output trees for `doc.docx` with three attachments:
+
+[source]
+----
+REGULAR, tika-server (one zip) REGULAR, tika-app -Z (loose)
+ 1.emf out/doc.docx.json
+ 2.zip out/doc.docx-embed/1.emf
+ 3.bin out/doc.docx-embed/2.zip
+ + with /all: 0.docx and one out/doc.docx-embed/3.bin
+ <name>.metadata.json per file
+
+FRICTIONLESS, tika-server (one zip) FRICTIONLESS, tika-app -Z
--unpack-format=FRICTIONLESS
+ datapackage.json out/doc.docx.json
+ metadata.json out/doc.docx/datapackage.json
+ unpacked/1.emf ... out/doc.docx/metadata.json
+ + with /all: unpacked/0.docx out/doc.docx/unpacked/1.emf ...
+----
+
+The top-level `doc.docx.json` on the command line is the normal pipes output
for the parse;
+the package's `metadata.json` carries the same rows again so it is
self-contained. Set
+`includeMetadata: false` in a `-c` file to drop it.
+
The `datapackage.json` file contains:
* List of all extracted files as "resources"
diff --git a/docs/modules/ROOT/pages/using-tika/cli/index.adoc
b/docs/modules/ROOT/pages/using-tika/cli/index.adoc
index 27670fad86..74fa7ad432 100644
--- a/docs/modules/ROOT/pages/using-tika/cli/index.adoc
+++ b/docs/modules/ROOT/pages/using-tika/cli/index.adoc
@@ -399,12 +399,22 @@ through every level. Both accept a single file or a
directory as input.
|Output format: `REGULAR` (default) or `FRICTIONLESS`.
|`--unpack-mode=<mode>`
-|Output mode: `ZIPPED` (default) or `DIRECTORY`.
+|Output mode: `ZIPPED` (default) or `DIRECTORY`. With
`--unpack-format=FRICTIONLESS` and no
+mode, `DIRECTORY`: the package is laid out under `<out>/<input>/`
(`datapackage.json`,
+`unpacked/...`) rather than zipped.
|`--unpack-include-metadata`
-|Include `metadata.json` in Frictionless output.
+|Metadata for every extracted file. A Frictionless package carries
`metadata.json` by default;
+this adds a `.metadata.json` sidecar per file to REGULAR zip output.
|===
+A `-c`/`--config` file's `unpack-config` is honoured and the flags above are
applied on top
+of it. That is the only way to get the source document itself into the output
+(`"unpack-config": {"includeOriginal": true}`); no flag does it. File naming
stays the
+CLI's own (`<input>-embed/00000001.<ext>`, detected suffixes) regardless of
the file's
+`suffixStrategy`, `zeroPadName`, `embeddedIdPrefix` or `keyBaseStrategy`. See
+xref:pipes/unpack-config.adoc#_what_each_surface_produces[what each surface
produces].
+
=== Examples
[source,bash]
diff --git a/docs/modules/ROOT/pages/using-tika/server/index.adoc
b/docs/modules/ROOT/pages/using-tika/server/index.adoc
index fd3d81dfac..db0bac2f68 100644
--- a/docs/modules/ROOT/pages/using-tika/server/index.adoc
+++ b/docs/modules/ROOT/pages/using-tika/server/index.adoc
@@ -83,7 +83,8 @@ xref:using-tika/grpc/index.adoc[Tika gRPC] for the gRPC
specifics.
=== `allowPerRequestConfig` — the `/config` endpoints
-`/tika/config`, `/tika/config/{text,html,xml,md,json}`, `/rmeta/config` and
`/meta/config` accept
+`/tika/config`, `/tika/config/{text,html,xml,md,json}`, `/rmeta/config`,
+`/rmeta/config/{handlerType}` and `/meta/config` accept
a caller-supplied parser configuration as a multipart `config` part. They do
not reveal the
server's own config, but a caller-supplied one can enable dangerous operations
such as pointing
parsers at attacker-chosen resources (see
@@ -158,6 +159,16 @@ Output format on the preset routes: an explicit format
segment (`/tika/preset/{n
always wins; without one, a `ContentHandlerFactory` the preset binds decides,
then the
config's, then the endpoint default (Markdown).
+Output format: name the handler once. A format segment sets it --
`/tika/config/xml`,
+`/rmeta/config/xml`, and `/unpack/all/{handlerType}` (or
`/unpack/all/config/{handlerType}`
+with a config part), which selects how `tk:content` is rendered in the
metadata `/all` adds
+(`/unpack/all/ignore` for none). Plain `/unpack` takes no segment: the
`metadata.json` a
+Frictionless package carries there uses the config's handler, or the default.
With no
+segment, a `content-handler-factory` in the
+`config` part does. Doing both is a `400`, whether or not the
+two agree -- on `/tika` the segment also fixes the route's `Content-Type`, so
silently
+preferring one would let the response lie about its own body.
+
On the batch surfaces, a `FetchEmitTuple` submitted to `/pipes` or `/async`
may carry a
top-level `"preset": "name"` field, resolved the same way beneath the tuple's
own
`parseContext`; an unknown name there answers the `PRESET_NOT_FOUND` status
(HTTP `400`).
@@ -326,6 +337,16 @@ curl -T document.docx http://localhost:9998/rmeta/markdown
# or /md
curl -T document.pdf http://localhost:9998/rmeta/ignore # metadata only
----
+POST with multipart for per-request configuration; the handler goes in the
path, not the
+`config` part:
+
+[source,bash]
+----
+curl -X POST http://localhost:9998/rmeta/config/xml \
+ -F "[email protected]" \
+ -F
"config={\"pdf-parser\":{\"ocr\":{\"strategy\":\"NO_OCR\"}}};type=application/json"
+----
+
=== Metadata only (`/meta`)
Returns container-document metadata only — no recursive embedded list, no
content. With no
diff --git a/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java
b/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java
index 97989e2c11..d4a50d196f 100644
--- a/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java
+++ b/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java
@@ -996,8 +996,11 @@ public class TikaCLI {
out.println(" --on-exists Behavior when an output
file exists: exception (default), replace, skip");
out.println(" -Z Recursively unpack all the
attachments, too");
out.println(" --unpack-format=<format> Output format: REGULAR
(default) or FRICTIONLESS");
- out.println(" --unpack-mode=<mode> Output mode: ZIPPED
(default) or DIRECTORY");
- out.println(" --unpack-include-metadata Include metadata.json in
Frictionless output");
+ out.println(" --unpack-mode=<mode> Output mode: ZIPPED
(default) or DIRECTORY;");
+ out.println(" DIRECTORY when
--unpack-format=FRICTIONLESS and no mode is given");
+ out.println(" --unpack-include-metadata Metadata for every
extracted file. A Frictionless package");
+ out.println(" carries metadata.json by
default; this adds per-file");
+ out.println(" sidecars to REGULAR zip
output");
out.println();
out.println();
}
diff --git a/tika-app/src/test/java/org/apache/tika/cli/TikaCLITest.java
b/tika-app/src/test/java/org/apache/tika/cli/TikaCLITest.java
index 90dd6ab5c8..b769ef63d0 100644
--- a/tika-app/src/test/java/org/apache/tika/cli/TikaCLITest.java
+++ b/tika-app/src/test/java/org/apache/tika/cli/TikaCLITest.java
@@ -36,6 +36,8 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.attribute.BasicFileAttributes;
+import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -689,6 +691,50 @@ public class TikaCLITest {
"Should have extracted embedded files in extractDir, got: " +
fileNames);
}
+ @Test
+ public void testFrictionlessWithoutModeIsADirectory() throws Exception {
+ Set<String> fileNames = unpack("-Z", "--unpack-format=FRICTIONLESS");
+ assertTrue(fileNames.stream().anyMatch(f ->
f.endsWith("/datapackage.json")),
+ "package should be laid out as a directory, got: " +
fileNames);
+ assertTrue(fileNames.stream().anyMatch(f -> f.contains("/unpacked/")),
fileNames.toString());
+ assertFalse(fileNames.stream().anyMatch(f -> f.endsWith(".zip") &&
!f.contains("/unpacked/")),
+ "no zip when no mode was asked for: " + fileNames);
+ assertTrue(fileNames.stream().anyMatch(f ->
f.endsWith("/metadata.json")),
+ "a Frictionless package carries metadata.json by default: " +
fileNames);
+ }
+
+ @Test
+ public void testFrictionlessIncludeMetadata() throws Exception {
+ Set<String> fileNames = unpack("-Z", "--unpack-format=FRICTIONLESS",
"--unpack-include-metadata");
+ String metadataJson = fileNames.stream().filter(f ->
f.endsWith("/metadata.json")).findFirst()
+ .orElseThrow(() -> new AssertionError("no metadata.json in " +
fileNames));
+ String content = Files.readString(extractDir.resolve(metadataJson));
+ assertTrue(content.contains("\"tk:embedded-depth\" : \"0\""),
+ "metadata.json should carry the container row: " + content);
+ assertTrue(content.contains("\"tk:content\""), content);
+ }
+
+ /** A -c file's unpack-config is honoured; it is the only way to get the
original back. */
+ @Test
+ public void testConfigFileUnpackConfigIsHonoured() throws Exception {
+ Set<String> fileNames = unpack("-Z", "--config=" + CONFIGS_DIR +
"/tika-config-unpack-original.json");
+ assertTrue(fileNames.stream().anyMatch(f ->
f.matches(".*-embed/0+\\.[^/]+")),
+ "includeOriginal from -c should add the container as
00000000.<ext>: " + fileNames);
+ assertTrue(fileNames.stream().anyMatch(f ->
f.matches(".*-embed/0+1\\.[^/]+")),
+ "-c must not change the CLI's file naming: " + fileNames);
+ }
+
+ private Set<String> unpack(String... flags) throws Exception {
+ Path input = Paths.get(new URI(resourcePrefix +
"/test_recursive_embedded.docx"));
+ List<String> params = new ArrayList<>(Arrays.asList(flags));
+ params.add("-p");
+ params.add(Paths.get("target/plugins").toAbsolutePath().toString());
+ params.add(input.toAbsolutePath().toString());
+ params.add(extractDir.toAbsolutePath().toString());
+ TikaCLI.main(params.toArray(new String[0]));
+ return getFileNames(extractDir);
+ }
+
@Test
public void testDefaultConfigException() throws Exception {
//default xml parser will throw TikaException
diff --git
a/tika-app/src/test/resources/configs/tika-config-unpack-original.json
b/tika-app/src/test/resources/configs/tika-config-unpack-original.json
new file mode 100644
index 0000000000..62e4bf9c33
--- /dev/null
+++ b/tika-app/src/test/resources/configs/tika-config-unpack-original.json
@@ -0,0 +1,7 @@
+{
+ "parse-context": {
+ "unpack-config": {
+ "includeOriginal": true
+ }
+ }
+}
diff --git
a/tika-pipes/tika-async-cli/src/main/java/org/apache/tika/async/cli/TikaAsyncCLI.java
b/tika-pipes/tika-async-cli/src/main/java/org/apache/tika/async/cli/TikaAsyncCLI.java
index eaccde098d..5b10176425 100644
---
a/tika-pipes/tika-async-cli/src/main/java/org/apache/tika/async/cli/TikaAsyncCLI.java
+++
b/tika-pipes/tika-async-cli/src/main/java/org/apache/tika/async/cli/TikaAsyncCLI.java
@@ -38,6 +38,7 @@ import org.slf4j.LoggerFactory;
import org.apache.tika.config.EmbeddedLimits;
import org.apache.tika.config.loader.TikaJsonConfig;
+import org.apache.tika.config.loader.TikaLoader;
import org.apache.tika.config.loader.TikaObjectMapperFactory;
import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.parser.ParseContext;
@@ -86,9 +87,11 @@ public class TikaAsyncCLI {
options.addOption(null, "unpack-format", true,
"output format for unpacking: REGULAR (default) or
FRICTIONLESS");
options.addOption(null, "unpack-mode", true,
- "output mode for unpacking: ZIPPED (default) or DIRECTORY");
+ "output mode for unpacking: ZIPPED (default) or DIRECTORY;
DIRECTORY when "
+ + "--unpack-format FRICTIONLESS is given with no
mode");
options.addOption(null, "unpack-include-metadata", false,
- "include metadata.json in Frictionless output");
+ "metadata for every extracted file: a Frictionless package
carries metadata.json "
+ + "by default; this adds per-file sidecars to REGULAR
zip output");
options.addOption(null, "on-exists", true,
"behavior when an output file already exists: exception
(default), replace or skip");
@@ -324,10 +327,11 @@ public class TikaAsyncCLI {
private static void processWithTikaConfig(PipesIterator pipesIterator,
Path tikaConfigPath, SimpleAsyncConfig asyncConfig) throws Exception {
long start = System.currentTimeMillis();
+ UnpackConfig configuredUnpack =
loadConfiguredUnpackConfig(tikaConfigPath, asyncConfig);
try (AsyncProcessor processor = AsyncProcessor.load(tikaConfigPath,
pipesIterator)) {
for (FetchEmitTuple t : pipesIterator) {
- configureExtractBytes(t, asyncConfig);
+ configureExtractBytes(t, asyncConfig, configuredUnpack);
configureHandler(t, asyncConfig);
boolean offered = processor.offer(t, TIMEOUT_MS);
if (!offered) {
@@ -357,7 +361,19 @@ public class TikaAsyncCLI {
t.getParseContext().set(ContentHandlerFactory.class, factory);
}
- private static void configureExtractBytes(FetchEmitTuple t,
SimpleAsyncConfig asyncConfig) {
+ /** The -c file's unpack-config; null when unpacking is off or none is
configured. */
+ private static UnpackConfig loadConfiguredUnpackConfig(Path tikaConfigPath,
+ SimpleAsyncConfig
asyncConfig)
+ throws TikaConfigException, IOException {
+ if (asyncConfig == null
+ || asyncConfig.getExtractBytesMode() ==
SimpleAsyncConfig.ExtractBytesMode.NONE) {
+ return null;
+ }
+ return
TikaLoader.load(tikaConfigPath).loadParseContext().get(UnpackConfig.class);
+ }
+
+ private static void configureExtractBytes(FetchEmitTuple t,
SimpleAsyncConfig asyncConfig,
+ UnpackConfig configured) {
if (asyncConfig == null) {
return;
}
@@ -382,23 +398,28 @@ public class TikaAsyncCLI {
}
// For RECURSIVE mode (-Z), use default unlimited depth
- UnpackConfig config = new UnpackConfig();
+ // Copied per tuple: the worker mutates it. File naming stays the
CLI's, config or not.
+ UnpackConfig config = configured != null ? configured.copy() : new
UnpackConfig();
config.setEmitter(TikaConfigAsyncWriter.EMITTER_NAME);
- config.setIncludeOriginal(false);
config.setSuffixStrategy(UnpackConfig.SUFFIX_STRATEGY.DETECTED);
config.setEmbeddedIdPrefix("-");
config.setZeroPadName(8);
config.setKeyBaseStrategy(UnpackConfig.KEY_BASE_STRATEGY.DEFAULT);
- // Apply Frictionless Data Package options
+ // Command-line flags win over the file
if (asyncConfig.getUnpackFormat() != null) {
config.setOutputFormat(UnpackConfig.OUTPUT_FORMAT.valueOf(asyncConfig.getUnpackFormat()));
+ // Loose files are what -z otherwise writes; a zip only when asked
for
+ if (asyncConfig.getUnpackMode() == null
+ && config.getOutputFormat() ==
UnpackConfig.OUTPUT_FORMAT.FRICTIONLESS) {
+ config.setOutputMode(UnpackConfig.OUTPUT_MODE.DIRECTORY);
+ }
}
if (asyncConfig.getUnpackMode() != null) {
config.setOutputMode(UnpackConfig.OUTPUT_MODE.valueOf(asyncConfig.getUnpackMode()));
}
if (asyncConfig.isUnpackIncludeMetadata()) {
- config.setIncludeFullMetadata(true);
+ config.setIncludeMetadata(true);
}
parseContext.set(UnpackConfig.class, config);
diff --git
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/UnpackConfig.java
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/UnpackConfig.java
index 208ae675f9..c235daf1a0 100644
---
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/UnpackConfig.java
+++
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/UnpackConfig.java
@@ -19,6 +19,12 @@ package org.apache.tika.pipes.core.extractor;
import java.io.Serializable;
import java.util.Objects;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.tika.annotation.TikaComponent;
@TikaComponent(name = "unpack-config")
@@ -29,6 +35,8 @@ public class UnpackConfig implements Serializable {
*/
private static final long serialVersionUID = -3861669115439125268L;
+ private static final Logger LOG =
LoggerFactory.getLogger(UnpackConfig.class);
+
/**
* Default maximum bytes to unpack per file: 10 GB.
* Use -1 to disable the limit (not recommended).
@@ -134,7 +142,8 @@ public class UnpackConfig implements Serializable {
// Zipping options
private boolean zipEmbeddedFiles = false;
- private boolean includeMetadataInZip = false;
+
+ private Boolean includeMetadata = null; // null: the format decides, see
writesMetadata()
// Maximum bytes to unpack per file (default 10GB, -1 to disable limit)
private long maxUnpackBytes = DEFAULT_MAX_UNPACK_BYTES;
@@ -142,7 +151,6 @@ public class UnpackConfig implements Serializable {
// Frictionless Data Package options
private OUTPUT_FORMAT outputFormat = OUTPUT_FORMAT.REGULAR;
private OUTPUT_MODE outputMode = OUTPUT_MODE.ZIPPED;
- private boolean includeFullMetadata = false; // Include metadata.json in
Frictionless output
/**
* Create an UnpackConfig with default settings.
@@ -150,6 +158,24 @@ public class UnpackConfig implements Serializable {
public UnpackConfig() {
}
+ /** The worker mutates its instance; hand out copies. */
+ public UnpackConfig copy() {
+ UnpackConfig c = new UnpackConfig();
+ c.zeroPadName = zeroPadName;
+ c.suffixStrategy = suffixStrategy;
+ c.embeddedIdPrefix = embeddedIdPrefix;
+ c.emitter = emitter;
+ c.includeOriginal = includeOriginal;
+ c.keyBaseStrategy = keyBaseStrategy;
+ c.emitKeyBase = emitKeyBase;
+ c.zipEmbeddedFiles = zipEmbeddedFiles;
+ c.includeMetadata = includeMetadata;
+ c.maxUnpackBytes = maxUnpackBytes;
+ c.outputFormat = outputFormat;
+ c.outputMode = outputMode;
+ return c;
+ }
+
public int getZeroPadName() {
return zeroPadName;
}
@@ -227,16 +253,41 @@ public class UnpackConfig implements Serializable {
this.zipEmbeddedFiles = zipEmbeddedFiles;
}
+ /** Null when the format decides; see {@link #writesMetadata()}. */
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ public Boolean getIncludeMetadata() {
+ return includeMetadata;
+ }
+
+ public void setIncludeMetadata(Boolean includeMetadata) {
+ this.includeMetadata = includeMetadata;
+ }
+
/**
- * Whether to include the metadata JSON for each embedded document in the
zip file.
- * Only applicable when {@link #isZipEmbeddedFiles()} is true.
+ * Metadata for every extracted file: {@code metadata.json} under
FRICTIONLESS, a sidecar
+ * per file under REGULAR zip output, nothing under REGULAR loose output.
Unset means on
+ * for FRICTIONLESS, off for REGULAR. Not a bean getter: the class is
introspected.
*/
+ public boolean writesMetadata() {
+ if (includeMetadata != null) {
+ return includeMetadata;
+ }
+ return outputFormat == OUTPUT_FORMAT.FRICTIONLESS;
+ }
+
+ /** @deprecated since 4.1, removal planned for 5.0; see {@link
#writesMetadata()} */
+ @Deprecated
+ @JsonIgnore
public boolean isIncludeMetadataInZip() {
- return includeMetadataInZip;
+ return writesMetadata();
}
+ /** @deprecated since 4.1, removal planned for 5.0; use {@link
#setIncludeMetadata(Boolean)} */
+ @Deprecated
+ @JsonProperty("includeMetadataInZip")
public void setIncludeMetadataInZip(boolean includeMetadataInZip) {
- this.includeMetadataInZip = includeMetadataInZip;
+ LOG.warn("unpack-config.includeMetadataInZip is deprecated (removal in
5.0); use includeMetadata");
+ setIncludeMetadata(includeMetadataInZip);
}
/**
@@ -302,16 +353,19 @@ public class UnpackConfig implements Serializable {
setOutputMode(OUTPUT_MODE.valueOf(outputMode));
}
- /**
- * Whether to include full RMETA-style metadata in metadata.json.
- * Only applicable when outputFormat is FRICTIONLESS.
- */
+ /** @deprecated since 4.1, removal planned for 5.0; see {@link
#writesMetadata()} */
+ @Deprecated
+ @JsonIgnore
public boolean isIncludeFullMetadata() {
- return includeFullMetadata;
+ return writesMetadata();
}
+ /** @deprecated since 4.1, removal planned for 5.0; use {@link
#setIncludeMetadata(Boolean)} */
+ @Deprecated
+ @JsonProperty("includeFullMetadata")
public void setIncludeFullMetadata(boolean includeFullMetadata) {
- this.includeFullMetadata = includeFullMetadata;
+ LOG.warn("unpack-config.includeFullMetadata is deprecated (removal in
5.0); use includeMetadata");
+ setIncludeMetadata(includeFullMetadata);
}
@Override
@@ -320,9 +374,9 @@ public class UnpackConfig implements Serializable {
suffixStrategy + ", embeddedIdPrefix='" + embeddedIdPrefix +
'\'' +
", emitter='" + emitter + '\'' + ", includeOriginal=" +
includeOriginal +
", keyBaseStrategy=" + keyBaseStrategy + ", emitKeyBase='" +
emitKeyBase + '\'' +
- ", zipEmbeddedFiles=" + zipEmbeddedFiles + ",
includeMetadataInZip=" + includeMetadataInZip +
+ ", zipEmbeddedFiles=" + zipEmbeddedFiles + ",
includeMetadata=" + includeMetadata +
", maxUnpackBytes=" + maxUnpackBytes + ", outputFormat=" +
outputFormat +
- ", outputMode=" + outputMode + ", includeFullMetadata=" +
includeFullMetadata + '}';
+ ", outputMode=" + outputMode + '}';
}
@Override
@@ -338,11 +392,10 @@ public class UnpackConfig implements Serializable {
keyBaseStrategy == config.keyBaseStrategy &&
Objects.equals(emitKeyBase, config.emitKeyBase) &&
zipEmbeddedFiles == config.zipEmbeddedFiles &&
- includeMetadataInZip == config.includeMetadataInZip &&
+ Objects.equals(includeMetadata, config.includeMetadata) &&
maxUnpackBytes == config.maxUnpackBytes &&
outputFormat == config.outputFormat &&
- outputMode == config.outputMode &&
- includeFullMetadata == config.includeFullMetadata;
+ outputMode == config.outputMode;
}
@Override
@@ -355,11 +408,10 @@ public class UnpackConfig implements Serializable {
result = 31 * result + Objects.hashCode(keyBaseStrategy);
result = 31 * result + Objects.hashCode(emitKeyBase);
result = 31 * result + Boolean.hashCode(zipEmbeddedFiles);
- result = 31 * result + Boolean.hashCode(includeMetadataInZip);
+ result = 31 * result + Objects.hashCode(includeMetadata);
result = 31 * result + Long.hashCode(maxUnpackBytes);
result = 31 * result + Objects.hashCode(outputFormat);
result = 31 * result + Objects.hashCode(outputMode);
- result = 31 * result + Boolean.hashCode(includeFullMetadata);
return result;
}
}
diff --git
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesWorker.java
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesWorker.java
index 8cc9962649..331d70d74e 100644
---
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesWorker.java
+++
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesWorker.java
@@ -377,7 +377,7 @@ class PipesWorker implements Callable<PipesResult> {
}
// Emit metadata.json if requested
- if (unpackConfig.isIncludeFullMetadata() && parseData != null &&
+ if (unpackConfig.writesMetadata() && parseData != null &&
parseData.getMetadataList() != null) {
String metadataEmitKey = baseEmitKey + "/metadata.json";
byte[] metadataBytes =
writeMetadataListAsJson(parseData.getMetadataList());
@@ -410,7 +410,7 @@ class PipesWorker implements Callable<PipesResult> {
zos.closeEntry();
// Add metadata.json if requested
- if (unpackConfig.isIncludeFullMetadata() && parseData != null &&
+ if (unpackConfig.writesMetadata() && parseData != null &&
parseData.getMetadataList() != null) {
ZipEntry metadataEntry = new ZipEntry("metadata.json");
zos.putNextEntry(metadataEntry);
@@ -479,7 +479,7 @@ class PipesWorker implements Callable<PipesResult> {
zos.closeEntry();
// Add metadata JSON if requested
- if (unpackConfig.isIncludeMetadataInZip()) {
+ if (unpackConfig.writesMetadata()) {
String metadataFileName = fileInfo.fileName() +
".metadata.json";
ZipEntry metadataEntry = new ZipEntry(metadataFileName);
zos.putNextEntry(metadataEntry);
diff --git
a/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/extractor/UnpackConfigDeprecatedFlagsTest.java
b/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/extractor/UnpackConfigDeprecatedFlagsTest.java
new file mode 100644
index 0000000000..0cdeeab749
--- /dev/null
+++
b/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/extractor/UnpackConfigDeprecatedFlagsTest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.core.extractor;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.jupiter.api.Test;
+
+import org.apache.tika.config.loader.TikaObjectMapperFactory;
+
+/** The strict mapper refuses unknown properties, so the pre-4.1 flag names
must keep loading. */
+public class UnpackConfigDeprecatedFlagsTest {
+
+ private static final ObjectMapper MAPPER =
TikaObjectMapperFactory.createMapper();
+
+ @Test
+ public void testIncludeMetadataInZipStillLoads() throws Exception {
+ UnpackConfig c = MAPPER.readValue("{\"includeMetadataInZip\": true}",
UnpackConfig.class);
+ assertTrue(c.writesMetadata());
+ assertTrue(c.isIncludeMetadataInZip());
+ }
+
+ @Test
+ public void testIncludeFullMetadataStillLoads() throws Exception {
+ UnpackConfig c = MAPPER.readValue("{\"includeFullMetadata\": true}",
UnpackConfig.class);
+ assertTrue(c.writesMetadata());
+ assertTrue(c.isIncludeFullMetadata());
+ }
+
+ @Test
+ public void testOnlyTheNewNameIsWritten() throws Exception {
+ UnpackConfig c = new UnpackConfig();
+ c.setIncludeMetadata(true);
+ String json = MAPPER.writeValueAsString(c);
+ assertTrue(json.contains("\"includeMetadata\":true"), json);
+ assertFalse(json.contains("includeMetadataInZip"), json);
+ assertFalse(json.contains("includeFullMetadata"), json);
+ assertTrue(MAPPER.readValue(json,
UnpackConfig.class).writesMetadata());
+ assertFalse(MAPPER.writeValueAsString(new
UnpackConfig()).contains("includeMetadata"),
+ "unset must not be written, or it would pin the format
default");
+ }
+
+ /** Unset: a Frictionless package carries metadata, REGULAR sidecars are
opt-in. */
+ @Test
+ public void testUnsetFollowsTheFormat() throws Exception {
+ UnpackConfig c = MAPPER.readValue("{\"outputFormat\":
\"FRICTIONLESS\"}", UnpackConfig.class);
+ assertTrue(c.writesMetadata());
+ assertFalse(MAPPER.readValue("{\"outputFormat\": \"REGULAR\"}",
UnpackConfig.class).writesMetadata());
+ assertFalse(new UnpackConfig().writesMetadata());
+
+ UnpackConfig optOut = MAPPER.readValue(
+ "{\"outputFormat\": \"FRICTIONLESS\", \"includeMetadata\":
false}", UnpackConfig.class);
+ assertFalse(optOut.writesMetadata(), "explicit false must beat the
format default");
+ }
+
+ @Test
+ public void testUnknownPropertyStillFails() {
+ assertThrows(JsonMappingException.class,
+ () -> MAPPER.readValue("{\"includeMetadatta\": true}",
UnpackConfig.class));
+ }
+
+ @Test
+ public void testCopyIsFieldForField() {
+ UnpackConfig c = new UnpackConfig();
+ c.setIncludeMetadata(true);
+ c.setIncludeOriginal(true);
+ c.setOutputFormat(UnpackConfig.OUTPUT_FORMAT.FRICTIONLESS);
+ c.setOutputMode(UnpackConfig.OUTPUT_MODE.DIRECTORY);
+ c.setEmitter("e");
+ UnpackConfig copy = c.copy();
+ assertEquals(c, copy);
+ copy.setIncludeMetadata(false);
+ assertTrue(c.writesMetadata(), "copy must not alias the original");
+ }
+}
diff --git
a/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/serialization/JsonFetchEmitTupleTest.java
b/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/serialization/JsonFetchEmitTupleTest.java
index abba24cb22..8dd5332fc1 100644
---
a/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/serialization/JsonFetchEmitTupleTest.java
+++
b/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/serialization/JsonFetchEmitTupleTest.java
@@ -20,6 +20,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.Reader;
import java.io.StringReader;
@@ -123,7 +124,7 @@ public class JsonFetchEmitTupleTest {
// Create UnpackConfig with specific settings
UnpackConfig unpackConfig = new UnpackConfig();
unpackConfig.setZipEmbeddedFiles(true);
- unpackConfig.setIncludeMetadataInZip(true);
+ unpackConfig.setIncludeMetadata(true);
unpackConfig.setEmitter("test-emitter");
unpackConfig.setSuffixStrategy(UnpackConfig.SUFFIX_STRATEGY.DETECTED);
parseContext.set(UnpackConfig.class, unpackConfig);
@@ -152,8 +153,10 @@ public class JsonFetchEmitTupleTest {
UnpackConfig deserializedConfig =
deserialized.getParseContext().get(UnpackConfig.class);
assertEquals(unpackConfig.isZipEmbeddedFiles(),
deserializedConfig.isZipEmbeddedFiles(),
"zipEmbeddedFiles should be preserved");
- assertEquals(unpackConfig.isIncludeMetadataInZip(),
deserializedConfig.isIncludeMetadataInZip(),
- "includeMetadataInZip should be preserved");
+ assertEquals(unpackConfig.getIncludeMetadata(),
deserializedConfig.getIncludeMetadata(),
+ "includeMetadata should be preserved");
+ assertTrue(deserializedConfig.isIncludeMetadataInZip(),
+ "deprecated getter should read the unified flag");
assertEquals(unpackConfig.getEmitter(),
deserializedConfig.getEmitter(),
"emitter should be preserved");
assertEquals(unpackConfig.getSuffixStrategy(),
deserializedConfig.getSuffixStrategy(),
diff --git
a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesParsingHelper.java
b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesParsingHelper.java
index 8d886b7a9b..1f295076be 100644
---
a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesParsingHelper.java
+++
b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesParsingHelper.java
@@ -657,7 +657,7 @@ public class PipesParsingHelper {
unpackConfig = new UnpackConfig();
}
- // Enable zip creation in the child process
+ // Enable zip creation in the child process (DIRECTORY was refused
before this)
unpackConfig.setZipEmbeddedFiles(true);
// Set suffix strategy to DETECTED so files get their proper
extensions (e.g., .wav, .jpg)
@@ -666,10 +666,10 @@ public class PipesParsingHelper {
// Set emitter to our file-system emitter
unpackConfig.setEmitter(UNPACK_EMITTER_ID);
- // Include original document if saveAll is requested
+ // /all = the original document plus metadata, in either output
format.
if (saveAll) {
unpackConfig.setIncludeOriginal(true);
- unpackConfig.setIncludeMetadataInZip(true);
+ unpackConfig.setIncludeMetadata(true);
}
parseContext.set(UnpackConfig.class, unpackConfig);
diff --git
a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/RecursiveMetadataResource.java
b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/RecursiveMetadataResource.java
index 4d8091d85e..3c82b50f73 100644
---
a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/RecursiveMetadataResource.java
+++
b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/RecursiveMetadataResource.java
@@ -103,7 +103,16 @@ public class RecursiveMetadataResource {
@POST
@Consumes("multipart/form-data")
@Produces({"application/json"})
- @Path("form{" + HANDLER_TYPE_PARAM + " : (\\w+)?}")
+ @Path("form")
+ public Response getMetadataFromMultipart(Attachment att) throws Exception {
+ return getMetadataFromMultipart(att, null);
+ }
+
+ /** As {@code /rmeta/form}, with the handler type in the path. */
+ @POST
+ @Consumes("multipart/form-data")
+ @Produces({"application/json"})
+ @Path("form/{" + HANDLER_TYPE_PARAM + "}")
public Response getMetadataFromMultipart(Attachment att,
@PathParam(HANDLER_TYPE_PARAM) String handlerTypeName) throws Exception {
try (TikaInputStream tis =
TikaInputStream.get(att.getObject(InputStream.class))) {
List<Metadata> metadataList = parseMetadata(tis,
tikaResource.newRequestMetadata(), att.getHeaders(),
@@ -124,6 +133,27 @@ public class RecursiveMetadataResource {
public Response getMetadataWithConfig(
List<Attachment> attachments,
@Context HttpHeaders httpHeaders) throws Exception {
+ return getMetadataWithConfig(attachments, httpHeaders, null);
+ }
+
+ /**
+ * As {@code /rmeta/config}, with the handler type in the path. One
templated route covers
+ * every format because /rmeta always produces JSON; /tika needs a route
per format only
+ * because its response media type changes with the handler.
+ * <p>
+ * As on {@code /tika/config/<format>}, naming a handler here and in the
config part too is
+ * a 400 rather than a silent pick.
+ *
+ * @param handlerTypeName content handler type: text, html, xml, body,
markdown, ignore
+ */
+ @POST
+ @Consumes("multipart/form-data")
+ @Produces({"application/json"})
+ @Path("config/{" + HANDLER_TYPE_PARAM + "}")
+ public Response getMetadataWithConfig(
+ List<Attachment> attachments,
+ @Context HttpHeaders httpHeaders,
+ @PathParam(HANDLER_TYPE_PARAM) String handlerTypeName) throws
Exception {
ParseContext context = tikaResource.createRequestContext();
Metadata metadata = tikaResource.newRequestMetadata();
@@ -131,8 +161,10 @@ public class RecursiveMetadataResource {
TikaResource.logRequest(LOG, "/rmeta/config", metadata);
+ tikaResource.applyExplicitFormat(context, handlerTypeName,
+ handlerTypeName != null && !handlerTypeName.isBlank());
return Response
- .ok(parseMetadataWithContext(tis, metadata, null, context))
+ .ok(parseMetadataWithContext(tis, metadata,
handlerTypeName, context))
.build();
}
}
diff --git
a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java
b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java
index 3b4e9b01d3..e9864d55fc 100644
---
a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java
+++
b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java
@@ -755,24 +755,50 @@ public class TikaResource {
// dangerous operations. try-with-resources in the helpers: the spooled
file part
// must be deleted even when config processing or the parse throws.
- private Response postConfigured(List<Attachment> attachments, String
handlerTypeName)
+ private Response postConfigured(List<Attachment> attachments, String
handlerTypeName,
+ boolean explicitFormat)
throws IOException, TikaConfigException {
ParseContext context = createRequestContext();
Metadata metadata = newRequestMetadata();
try (TikaInputStream tis = setupMultipartConfig(attachments, metadata,
context)) {
+ applyExplicitFormat(context, handlerTypeName, explicitFormat);
return produceRawOutput(tis, metadata, context, handlerTypeName);
}
}
- private Metadata postConfiguredJson(List<Attachment> attachments, String
handlerTypeName)
+ private Metadata postConfiguredJson(List<Attachment> attachments, String
handlerTypeName,
+ boolean explicitFormat)
throws IOException, TikaConfigException {
ParseContext context = createRequestContext();
Metadata metadata = newRequestMetadata();
try (TikaInputStream tis = setupMultipartConfig(attachments, metadata,
context)) {
+ applyExplicitFormat(context, handlerTypeName, explicitFormat);
return produceJson(tis, metadata, context, handlerTypeName);
}
}
+ /**
+ * Binds the format named by an explicit path segment, refusing the
request if the config part
+ * named a handler too. Silently picking either one is worse than a 400:
the route's
+ * {@code @Produces} is fixed by the segment, so letting the config part
win makes
+ * /tika/config/xml answer text/xml with a markdown body. Any factory in
the context at this
+ * point came from the config part -- createRequestContext carries request
deltas only.
+ */
+ public void applyExplicitFormat(ParseContext context, String
handlerTypeName,
+ boolean explicitFormat) {
+ if (!explicitFormat) {
+ return;
+ }
+ if (context.get(ContentHandlerFactory.class) != null) {
+ throw new BadRequestException(
+ "The path names a content handler ('" + handlerTypeName +
"') and so does the "
+ + "config part. Use one or the other: drop
content-handler-factory "
+ + "from the config part, or post to the endpoint
with no format "
+ + "segment.");
+ }
+ setupContentHandlerFactory(context, handlerTypeName);
+ }
+
/** Multipart document with optional config; returns Markdown unless the
config names a handler. */
@POST
@Consumes("multipart/form-data")
@@ -780,7 +806,7 @@ public class TikaResource {
@Path("config")
public Response postRaw(List<Attachment> attachments, @Context HttpHeaders
httpHeaders)
throws IOException, TikaConfigException {
- return postConfigured(attachments, "md");
+ return postConfigured(attachments, "md", false);
}
/** Multipart document with optional config; returns body-only plain text.
*/
@@ -790,7 +816,7 @@ public class TikaResource {
@Path("config/text")
public Response postText(List<Attachment> attachments, @Context
HttpHeaders httpHeaders)
throws IOException, TikaConfigException {
- return postConfigured(attachments, "body");
+ return postConfigured(attachments, "body", true);
}
/** Multipart document with optional config; returns HTML. */
@@ -800,7 +826,7 @@ public class TikaResource {
@Path("config/html")
public Response postHtml(List<Attachment> attachments, @Context
HttpHeaders httpHeaders)
throws IOException, TikaConfigException {
- return postConfigured(attachments, "html");
+ return postConfigured(attachments, "html", true);
}
/** Multipart document with optional config; returns XML. */
@@ -810,7 +836,7 @@ public class TikaResource {
@Path("config/xml")
public Response postXml(List<Attachment> attachments, @Context HttpHeaders
httpHeaders)
throws IOException, TikaConfigException {
- return postConfigured(attachments, "xml");
+ return postConfigured(attachments, "xml", true);
}
/** Multipart document with optional config; returns Markdown. */
@@ -820,7 +846,7 @@ public class TikaResource {
@Path("config/md")
public Response postMarkdown(List<Attachment> attachments, @Context
HttpHeaders httpHeaders)
throws IOException, TikaConfigException {
- return postConfigured(attachments, "md");
+ return postConfigured(attachments, "md", true);
}
/** Multipart document with optional config; returns JSON with the default
(markdown) handler. */
@@ -830,7 +856,7 @@ public class TikaResource {
@Path("config/json")
public Metadata postJson(List<Attachment> attachments, @Context
HttpHeaders httpHeaders)
throws IOException, TikaConfigException {
- return postConfiguredJson(attachments, null);
+ return postConfiguredJson(attachments, null, false);
}
/**
@@ -847,7 +873,7 @@ public class TikaResource {
public Metadata postJsonWithHandler(List<Attachment> attachments, @Context
HttpHeaders httpHeaders,
@PathParam(HANDLER_TYPE_PARAM) String
handlerTypeName)
throws IOException, TikaConfigException {
- return postConfiguredJson(attachments, handlerTypeName);
+ return postConfiguredJson(attachments, handlerTypeName, true);
}
// ==================== Internal methods ====================
diff --git
a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/UnpackerResource.java
b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/UnpackerResource.java
index f47243795e..321d9673ab 100644
---
a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/UnpackerResource.java
+++
b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/UnpackerResource.java
@@ -32,7 +32,6 @@ import jakarta.ws.rs.core.Context;
import jakarta.ws.rs.core.HttpHeaders;
import jakarta.ws.rs.core.Response;
import jakarta.ws.rs.core.StreamingOutput;
-import jakarta.ws.rs.core.UriInfo;
import org.apache.cxf.jaxrs.ext.multipart.Attachment;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -100,7 +99,7 @@ import org.apache.tika.pipes.core.extractor.UnpackConfig;
* "unpack-config": {
* "outputFormat": "FRICTIONLESS",
* "outputMode": "ZIPPED",
- * "includeFullMetadata": true
+ * "includeMetadata": true
* }
* }
* }
@@ -110,7 +109,7 @@ import org.apache.tika.pipes.core.extractor.UnpackConfig;
* <pre>
* output.zip
* ├── datapackage.json # Manifest with file list, SHA256 hashes,
mimetypes
- * ├── metadata.json # Full RMETA metadata (if
includeFullMetadata=true)
+ * ├── metadata.json # Full RMETA metadata (unless
includeMetadata=false)
* └── unpacked/
* ├── 00000001.pdf
* ├── 00000002.png
@@ -130,10 +129,20 @@ public class UnpackerResource {
private static final Logger LOG =
LoggerFactory.getLogger(UnpackerResource.class);
+ private static final String HANDLER_TYPE_PARAM = "handler";
+
private final TikaResource tikaResource;
public UnpackerResource(TikaResource tikaResource) {
this.tikaResource = tikaResource;
+ // DIRECTORY can never serve /unpack, which returns one zip: fail at
startup
+ UnpackConfig configured = tikaResource.newConfigUnpackConfig();
+ if (configured != null && configured.getOutputMode() ==
UnpackConfig.OUTPUT_MODE.DIRECTORY) {
+ throw new IllegalStateException(
+ "unpack-config.outputMode DIRECTORY cannot serve /unpack,
which returns one "
+ + "zip. Set ZIPPED, or disable the unpack endpoint
and use /pipes or "
+ + "/async for per-file emission.");
+ }
}
/**
@@ -145,21 +154,10 @@ public class UnpackerResource {
* @param info URI info
* @return streaming zip response
*/
- // The wildcard id would otherwise absorb a transposed preset URL
(/unpack/all/preset/x)
- // and silently run with no preset applied.
- private static void rejectPresetInWildcard(UriInfo info) {
- String id = info.getPathParameters().getFirst("id");
- if (id != null && (id.equals("/preset") || id.startsWith("/preset/")))
{
- throw new jakarta.ws.rs.NotFoundException(
- "preset routes are PUT /unpack/preset/{name}[/all]");
- }
- }
-
- @jakarta.ws.rs.Path("/{id:(/.*)?}")
+ @jakarta.ws.rs.Path("")
@PUT
@Produces("application/zip")
- public Response unpack(InputStream is, @Context HttpHeaders httpHeaders,
@Context UriInfo info) throws Exception {
- rejectPresetInWildcard(info);
+ public Response unpack(InputStream is, @Context HttpHeaders httpHeaders)
throws Exception {
ParseContext pc = tikaResource.createRequestContext();
Metadata metadata = tikaResource.newRequestMetadata();
try (TikaInputStream tis = TikaInputStream.get(is)) {
@@ -178,15 +176,16 @@ public class UnpackerResource {
* @param info URI info
* @return streaming zip response
*/
- @jakarta.ws.rs.Path("/{id:(/.*)?}")
+ @jakarta.ws.rs.Path("")
@POST
@Consumes("multipart/form-data")
@Produces("application/zip")
- public Response unpackWithConfig(List<Attachment> attachments, @Context
HttpHeaders httpHeaders, @Context UriInfo info) throws Exception {
- rejectPresetInWildcard(info);
+ public Response unpackWithConfig(List<Attachment> attachments,
+ @Context HttpHeaders httpHeaders) throws
Exception {
ParseContext pc = tikaResource.createRequestContext();
Metadata metadata = tikaResource.newRequestMetadata();
try (TikaInputStream tis =
tikaResource.setupMultipartConfig(attachments, metadata, pc)) {
+ rejectDirectoryMode(pc);
TikaResource.logRequest(LOG, "/unpack", metadata);
return doUnpack(tis, metadata, pc, false);
}
@@ -201,12 +200,22 @@ public class UnpackerResource {
* @param info URI info
* @return streaming zip response
*/
- @jakarta.ws.rs.Path("/all{id:(/.*)?}")
+ @jakarta.ws.rs.Path("/all")
+ @PUT
+ @Produces("application/zip")
+ public Response unpackAll(InputStream is, @Context HttpHeaders
httpHeaders) throws Exception {
+ return unpackAll(is, httpHeaders, null);
+ }
+
+ /** As {@code /unpack/all}, with the handler for the metadata's {@code
tk:content} in the path. */
+ @jakarta.ws.rs.Path("/all/{" + HANDLER_TYPE_PARAM + "}")
@PUT
@Produces("application/zip")
- public Response unpackAll(InputStream is, @Context HttpHeaders
httpHeaders, @Context UriInfo info) throws Exception {
- rejectPresetInWildcard(info);
+ public Response unpackAll(InputStream is, @Context HttpHeaders httpHeaders,
+ @jakarta.ws.rs.PathParam(HANDLER_TYPE_PARAM)
String handlerTypeName)
+ throws Exception {
ParseContext pc = tikaResource.createRequestContext();
+ applyHandler(pc, handlerTypeName);
Metadata metadata = tikaResource.newRequestMetadata();
try (TikaInputStream tis = TikaInputStream.get(is)) {
fillMetadata(null, metadata, httpHeaders.getRequestHeaders());
@@ -224,15 +233,53 @@ public class UnpackerResource {
* @param info URI info
* @return streaming zip response
*/
- @jakarta.ws.rs.Path("/all{id:(/.*)?}")
+ @jakarta.ws.rs.Path("/all")
+ @POST
+ @Consumes("multipart/form-data")
+ @Produces("application/zip")
+ public Response unpackAllWithConfig(List<Attachment> attachments,
+ @Context HttpHeaders httpHeaders)
throws Exception {
+ return unpackAllWithConfig(attachments, httpHeaders, null);
+ }
+
+ /**
+ * The config-variant spelling, {@code POST /unpack/all/config}: the same
as
+ * {@code POST /unpack/all}. 4.0 accepted it by accident (the wildcard
route swallowed the
+ * segment) and documented it; kept so it does not turn into a handler
named "config".
+ */
+ @jakarta.ws.rs.Path("/all/config")
+ @POST
+ @Consumes("multipart/form-data")
+ @Produces("application/zip")
+ public Response unpackAllConfig(List<Attachment> attachments, @Context
HttpHeaders httpHeaders)
+ throws Exception {
+ return unpackAllWithConfig(attachments, httpHeaders, null);
+ }
+
+ /** As {@code POST /unpack/all/config}, with the sidecar handler named in
the path, like {@code /rmeta/config/{handlerType}}. */
+ @jakarta.ws.rs.Path("/all/config/{" + HANDLER_TYPE_PARAM + "}")
+ @POST
+ @Consumes("multipart/form-data")
+ @Produces("application/zip")
+ public Response unpackAllConfig(List<Attachment> attachments, @Context
HttpHeaders httpHeaders,
+
@jakarta.ws.rs.PathParam(HANDLER_TYPE_PARAM) String handlerTypeName)
+ throws Exception {
+ return unpackAllWithConfig(attachments, httpHeaders, handlerTypeName);
+ }
+
+ /** As {@code POST /unpack/all}, with the sidecar handler named in the
path. */
+ @jakarta.ws.rs.Path("/all/{" + HANDLER_TYPE_PARAM + "}")
@POST
@Consumes("multipart/form-data")
@Produces("application/zip")
- public Response unpackAllWithConfig(List<Attachment> attachments, @Context
HttpHeaders httpHeaders, @Context UriInfo info) throws Exception {
- rejectPresetInWildcard(info);
+ public Response unpackAllWithConfig(List<Attachment> attachments, @Context
HttpHeaders httpHeaders,
+
@jakarta.ws.rs.PathParam(HANDLER_TYPE_PARAM) String handlerTypeName)
+ throws Exception {
ParseContext pc = tikaResource.createRequestContext();
Metadata metadata = tikaResource.newRequestMetadata();
try (TikaInputStream tis =
tikaResource.setupMultipartConfig(attachments, metadata, pc)) {
+ rejectDirectoryMode(pc);
+ applyHandler(pc, handlerTypeName);
TikaResource.logRequest(LOG, "/unpack/all", metadata);
return doUnpack(tis, metadata, pc, true);
}
@@ -249,6 +296,7 @@ public class UnpackerResource {
@jakarta.ws.rs.PathParam("presetName")
String presetName)
throws Exception {
ParseContext pc = tikaResource.createPresetContext(presetName);
+ rejectDirectoryMode(pc);
Metadata metadata = tikaResource.newRequestMetadata();
try (TikaInputStream tis = TikaInputStream.get(is)) {
fillMetadata(null, metadata, httpHeaders.getRequestHeaders());
@@ -267,6 +315,7 @@ public class UnpackerResource {
@jakarta.ws.rs.PathParam("presetName")
String presetName)
throws Exception {
ParseContext pc = tikaResource.createPresetContext(presetName);
+ rejectDirectoryMode(pc);
Metadata metadata = tikaResource.newRequestMetadata();
try (TikaInputStream tis = TikaInputStream.get(is)) {
fillMetadata(null, metadata, httpHeaders.getRequestHeaders());
@@ -275,6 +324,27 @@ public class UnpackerResource {
}
}
+ /** A request asking for DIRECTORY contradicts the one-zip transport: 400,
not a silent pin. */
+ static void rejectDirectoryMode(ParseContext pc) {
+ UnpackConfig requested = pc.get(UnpackConfig.class);
+ if (requested != null && requested.getOutputMode() ==
UnpackConfig.OUTPUT_MODE.DIRECTORY) {
+ throw new jakarta.ws.rs.BadRequestException(
+ "unpack-config.outputMode DIRECTORY emits each file as its
own item; /unpack "
+ + "returns one zip. Omit outputMode here, or use
/pipes or /async.");
+ }
+ }
+
+ /** Same rule as /tika and /rmeta: a handler named in the path and the
config part is a 400. */
+ private void applyHandler(ParseContext pc, String handlerTypeName) {
+ // a truncated preset URL parses as a handler; point at the real route
instead
+ if ("preset".equals(handlerTypeName)) {
+ throw new jakarta.ws.rs.NotFoundException(
+ "preset routes are PUT /unpack/preset/{name}[/all]");
+ }
+ tikaResource.applyExplicitFormat(pc, handlerTypeName,
+ handlerTypeName != null && !handlerTypeName.isBlank());
+ }
+
/**
* Core unpack logic using pipes-based parsing.
* The child process creates the zip file, and we stream it directly back.
diff --git
a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/RecursiveMetadataConfigHandlerTest.java
b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/RecursiveMetadataConfigHandlerTest.java
new file mode 100644
index 0000000000..1dfd8c5bb4
--- /dev/null
+++
b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/RecursiveMetadataConfigHandlerTest.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.server.core;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.util.ArrayList;
+import java.util.List;
+
+import jakarta.ws.rs.core.Response;
+import org.apache.cxf.jaxrs.JAXRSServerFactoryBean;
+import org.apache.cxf.jaxrs.client.WebClient;
+import org.apache.cxf.jaxrs.ext.multipart.Attachment;
+import org.apache.cxf.jaxrs.ext.multipart.ContentDisposition;
+import org.apache.cxf.jaxrs.ext.multipart.MultipartBody;
+import org.apache.cxf.jaxrs.lifecycle.SingletonResourceProvider;
+import org.junit.jupiter.api.Test;
+
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.serialization.JsonMetadataList;
+import org.apache.tika.server.core.resource.RecursiveMetadataResource;
+import org.apache.tika.server.core.writer.MetadataListMessageBodyWriter;
+
+/** Handler selection on /rmeta's multipart config endpoint: by path segment
and by config part. */
+public class RecursiveMetadataConfigHandlerTest extends CXFTestBase {
+
+ private static final String META_PATH = "/rmeta";
+ private static final String TEST_DOC =
"test-documents/mock/hello_world.xml";
+ private static final String XML_FACTORY =
+ "{\"basic-content-handler-factory\": {\"type\": \"XML\"}}";
+
+ @Override
+ protected boolean isAllowPerRequestConfig() {
+ return true;
+ }
+
+ @Override
+ protected void setUpResources(JAXRSServerFactoryBean sf) {
+ sf.setResourceClasses(RecursiveMetadataResource.class);
+ sf.setResourceProvider(RecursiveMetadataResource.class,
+ new SingletonResourceProvider(new
RecursiveMetadataResource(tikaResource)));
+ }
+
+ @Override
+ protected void setUpProviders(JAXRSServerFactoryBean sf) {
+ List<Object> providers = new ArrayList<>();
+ providers.add(new MetadataListMessageBodyWriter());
+ providers.add(new TikaServerParseExceptionMapper());
+ providers.add(new BadRequestExceptionMapper());
+ sf.setProviders(providers);
+ }
+
+ @Test
+ public void testConfigPartDefaultsToMarkdown() throws Exception {
+ String content = rmetaConfigContent(null, null);
+ assertTrue(content.contains("hello world"), content);
+ assertFalse(content.contains("<p>"), "default should be markdown, was:
" + content);
+ }
+
+ @Test
+ public void testConfigPartCanSelectXmlHandler() throws Exception {
+ String content = rmetaConfigContent(null, XML_FACTORY);
+ assertTrue(content.contains("<p>hello world</p>"),
+ "config part should be able to select the XML handler, was: "
+ content);
+ }
+
+ @Test
+ public void testPathSegmentCanSelectXmlHandler() throws Exception {
+ String content = rmetaConfigContent("xml", null);
+ assertTrue(content.contains("<p>hello world</p>"),
+ "config/<handler> should select the XML handler, was: " +
content);
+ }
+
+ /** A config part that names no handler is not a conflict. */
+ @Test
+ public void testPathSegmentAppliesAlongsideAConfigPart() throws Exception {
+ String content = rmetaConfigContent("xml", "{\"output-limits\":
{\"writeLimit\": 100}}");
+ assertTrue(content.contains("<p>hello world</p>"),
+ "config/<handler> should still apply when a config part is
present, was: " + content);
+ }
+
+ /** Naming the handler in the path and in the config part is a 400, not a
silent pick. */
+ @Test
+ public void testPathSegmentPlusConfigFactoryIsBadRequest() throws
Exception {
+ assertEquals(400, rmetaConfigResponse("markdown",
XML_FACTORY).getStatus());
+ assertEquals(400, rmetaConfigResponse("xml", XML_FACTORY).getStatus());
+ }
+
+ @Test
+ public void testUnrecognizedHandlerIsBadRequest() throws Exception {
+ assertEquals(400, rmetaConfigResponse("somethingOrOther",
null).getStatus());
+ }
+
+ @Test
+ public void testFormTakesHandlerAsItsOwnPathSegment() throws Exception {
+ assertTrue(formContent("/xml").contains("<p>hello world</p>"));
+ assertFalse(formContent("").contains("<p>"));
+ }
+
+ /** TIKA-1716 built /rmeta/form{handler} as one segment, which also
matched /rmeta/formxml. */
+ @Test
+ public void testFormHandlerIsNotAnUndelimitedSuffix() throws Exception {
+ assertEquals(405, formResponse("xml").getStatus());
+ }
+
+ private String formContent(String handlerSuffix) throws Exception {
+ Response response = formResponse(handlerSuffix);
+ assertEquals(200, response.getStatus());
+ Reader reader = new InputStreamReader((InputStream)
response.getEntity(), UTF_8);
+ return
JsonMetadataList.fromJson(reader).get(0).get(TikaCoreProperties.TIKA_CONTENT);
+ }
+
+ private Response formResponse(String suffix) throws Exception {
+ ContentDisposition fileCd =
+ new ContentDisposition("form-data; name=\"file\";
filename=\"hello_world.xml\"");
+ Attachment fileAtt = new Attachment("file",
+ ClassLoader.getSystemResourceAsStream(TEST_DOC), fileCd);
+ return WebClient
+ .create(endPoint + META_PATH + "/form" + suffix)
+ .type("multipart/form-data")
+ .accept("application/json")
+ .post(fileAtt);
+ }
+
+ private String rmetaConfigContent(String handlerSuffix, String configJson)
throws Exception {
+ Response response = rmetaConfigResponse(handlerSuffix, configJson);
+ assertEquals(200, response.getStatus());
+ Reader reader = new InputStreamReader((InputStream)
response.getEntity(), UTF_8);
+ List<Metadata> metadataList = JsonMetadataList.fromJson(reader);
+ return metadataList.get(0).get(TikaCoreProperties.TIKA_CONTENT);
+ }
+
+ private Response rmetaConfigResponse(String handlerSuffix, String
configJson) throws Exception {
+ ContentDisposition fileCd =
+ new ContentDisposition("form-data; name=\"file\";
filename=\"hello_world.xml\"");
+ Attachment fileAtt = new Attachment("file",
+ ClassLoader.getSystemResourceAsStream(TEST_DOC), fileCd);
+ List<Attachment> atts = new ArrayList<>();
+ atts.add(fileAtt);
+ if (configJson != null) {
+ atts.add(new Attachment("config", "application/json",
+ new ByteArrayInputStream(configJson.getBytes(UTF_8))));
+ }
+
+ String path = META_PATH + "/config" + (handlerSuffix == null ? "" :
"/" + handlerSuffix);
+ return WebClient
+ .create(endPoint + path)
+ .type("multipart/form-data")
+ .accept("application/json")
+ .post(new MultipartBody(atts));
+ }
+}
diff --git
a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaResourceConfigFormatTest.java
b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaResourceConfigFormatTest.java
new file mode 100644
index 0000000000..8c05d9e143
--- /dev/null
+++
b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaResourceConfigFormatTest.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.server.core;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+import jakarta.ws.rs.core.Response;
+import org.apache.cxf.jaxrs.JAXRSServerFactoryBean;
+import org.apache.cxf.jaxrs.client.WebClient;
+import org.apache.cxf.jaxrs.ext.multipart.Attachment;
+import org.apache.cxf.jaxrs.ext.multipart.ContentDisposition;
+import org.apache.cxf.jaxrs.ext.multipart.MultipartBody;
+import org.apache.cxf.jaxrs.lifecycle.SingletonResourceProvider;
+import org.junit.jupiter.api.Test;
+
+import org.apache.tika.server.core.resource.TikaResource;
+import org.apache.tika.server.core.writer.JSONMessageBodyWriter;
+
+/**
+ * A format segment on /tika/config/<format> fixes the route's {@code
@Produces}, so a
+ * config part must not be able to change the body's format out from under it.
+ */
+public class TikaResourceConfigFormatTest extends CXFTestBase {
+
+ private static final String TEST_DOC =
"test-documents/mock/hello_world.xml";
+ private static final String MARKDOWN_FACTORY =
+ "{\"basic-content-handler-factory\": {\"type\": \"MARKDOWN\"}}";
+ private static final String XML_FACTORY =
+ "{\"basic-content-handler-factory\": {\"type\": \"XML\"}}";
+
+ @Override
+ protected boolean isAllowPerRequestConfig() {
+ return true;
+ }
+
+ @Override
+ protected void setUpResources(JAXRSServerFactoryBean sf) {
+ sf.setResourceClasses(TikaResource.class);
+ sf.setResourceProvider(TikaResource.class, new
SingletonResourceProvider(tikaResource));
+ }
+
+ @Override
+ protected void setUpProviders(JAXRSServerFactoryBean sf) {
+ List<Object> providers = new ArrayList<>();
+ providers.add(new TikaServerParseExceptionMapper());
+ providers.add(new BadRequestExceptionMapper());
+ providers.add(new JSONMessageBodyWriter());
+ sf.setProviders(providers);
+ }
+
+ @Test
+ public void testFormatSegmentPlusConfigFactoryIsBadRequest() throws
Exception {
+ assertEquals(400, post("/tika/config/xml",
MARKDOWN_FACTORY).getStatus());
+ }
+
+ /** Agreeing is still a conflict: the rule is syntactic, so callers name
the handler once. */
+ @Test
+ public void testAgreeingSegmentAndConfigFactoryIsAlsoBadRequest() throws
Exception {
+ assertEquals(400, post("/tika/config/xml", XML_FACTORY).getStatus());
+ }
+
+ @Test
+ public void testFormatSegmentAloneStillSetsTheFormat() throws Exception {
+ Response response = post("/tika/config/xml", null);
+ assertEquals("text/xml;charset=utf-8",
response.getHeaderString("Content-Type"));
+ assertTrue(body(response).startsWith("<html"));
+ assertFalse(body(post("/tika/config/md", null)).contains("<p>"));
+ }
+
+ /** No format segment: the config part is the only thing naming a handler,
so it applies. */
+ @Test
+ public void testBareConfigStillHonorsTheConfigPartFactory() throws
Exception {
+ assertTrue(body(post("/tika/config",
XML_FACTORY)).startsWith("<html"));
+ }
+
+ private Response post(String path, String configJson) throws Exception {
+ ContentDisposition cd =
+ new ContentDisposition("form-data; name=\"file\";
filename=\"hello_world.xml\"");
+ List<Attachment> atts = new ArrayList<>();
+ atts.add(new Attachment("file",
ClassLoader.getSystemResourceAsStream(TEST_DOC), cd));
+ if (configJson != null) {
+ atts.add(new Attachment("config", "application/json",
+ new ByteArrayInputStream(configJson.getBytes(UTF_8))));
+ }
+ return WebClient.create(endPoint + path)
+ .type("multipart/form-data")
+ .post(new MultipartBody(atts));
+ }
+
+ private String body(Response response) throws Exception {
+ assertEquals(200, response.getStatus());
+ return getStringFromInputStream((InputStream)
response.getEntity()).trim();
+ }
+}
diff --git
a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/resource/RequestContextIsolationTest.java
b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/resource/RequestContextIsolationTest.java
index 2c6bb1351a..3d2a5f13a0 100644
---
a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/resource/RequestContextIsolationTest.java
+++
b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/resource/RequestContextIsolationTest.java
@@ -21,6 +21,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNotSame;
import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.nio.file.Files;
@@ -117,6 +118,20 @@ public class RequestContextIsolationTest {
assertEquals(BasicContentHandlerFactory.HANDLER_TYPE.TEXT,
chf.getType());
}
+ /** The server's own DIRECTORY unpack-config can never serve /unpack:
refuse at startup. */
+ @Test
+ public void directoryUnpackConfigFailsAtStartup() throws Exception {
+ TikaResource directory = newTikaResource("""
+ { "parse-context": { "unpack-config": { "outputMode":
"DIRECTORY" } } }
+ """);
+ assertThrows(IllegalStateException.class, () -> new
UnpackerResource(directory));
+
+ new UnpackerResource(newTikaResource("""
+ { "parse-context": { "unpack-config": { "outputMode": "ZIPPED"
} } }
+ """));
+ new UnpackerResource(newTikaResource("{}"));
+ }
+
/** The write limiter no longer rides in the context, so it must be
applied to the metadata. */
@Test
public void configuredMetadataLimiterStillBoundsRequestMetadata() throws
Exception {
diff --git
a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/RecursiveMetadataResourceTest.java
b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/RecursiveMetadataResourceTest.java
index 9d8523a7bc..3a50b321ed 100644
---
a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/RecursiveMetadataResourceTest.java
+++
b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/RecursiveMetadataResourceTest.java
@@ -195,7 +195,6 @@ public class RecursiveMetadataResourceTest extends
CXFTestBase {
}
@Test
- @org.junit.jupiter.api.Disabled("multipart config endpoint not yet fully
supported with pipes-based parsing")
public void testPasswordProtectedWithConfig() throws Exception {
// Test with password via JSON config
String configJson = """
diff --git
a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackFrictionlessTest.java
b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackFrictionlessTest.java
new file mode 100644
index 0000000000..511ea75c7d
--- /dev/null
+++
b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackFrictionlessTest.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.server.standard;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import jakarta.ws.rs.core.Response;
+import org.apache.cxf.jaxrs.JAXRSServerFactoryBean;
+import org.apache.cxf.jaxrs.client.WebClient;
+import org.apache.cxf.jaxrs.ext.multipart.Attachment;
+import org.apache.cxf.jaxrs.ext.multipart.ContentDisposition;
+import org.apache.cxf.jaxrs.ext.multipart.MultipartBody;
+import org.apache.cxf.jaxrs.lifecycle.SingletonResourceProvider;
+import org.junit.jupiter.api.Test;
+
+import org.apache.tika.serialization.config.JsonConfigHelper;
+import org.apache.tika.server.core.CXFTestBase;
+import org.apache.tika.server.core.TikaServerParseExceptionMapper;
+import org.apache.tika.server.core.resource.UnpackerResource;
+import org.apache.tika.server.core.writer.TarWriter;
+import org.apache.tika.server.core.writer.ZipWriter;
+
+/** /unpack under a server-level Frictionless config, with no per-request
config. */
+public class UnpackFrictionlessTest extends CXFTestBase {
+
+ private static final String TEST_DOC =
"test-documents/test_recursive_embedded.docx";
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
+ private static final String SERVER_CONFIG = """
+ {
+ "parsers": [ { "default-parser": {} } ],
+ "parse-context": {
+ "unpack-config": { "outputFormat": "FRICTIONLESS",
"outputMode": "ZIPPED" }
+ }
+ }
+ """;
+
+ private Path unpackTempDir;
+
+ @Override
+ protected boolean isAllowPerRequestConfig() {
+ return true;
+ }
+
+ @Override
+ protected InputStream getTikaConfigInputStream() throws IOException {
+ return new
ByteArrayInputStream(SERVER_CONFIG.getBytes(StandardCharsets.UTF_8));
+ }
+
+ @Override
+ protected void setUpResources(JAXRSServerFactoryBean sf) {
+ sf.setResourceClasses(UnpackerResource.class);
+ sf.setResourceProvider(UnpackerResource.class,
+ new SingletonResourceProvider(new
UnpackerResource(tikaResource)));
+ }
+
+ @Override
+ protected void setUpProviders(JAXRSServerFactoryBean sf) {
+ List<Object> providers = new ArrayList<>();
+ providers.add(new TarWriter());
+ providers.add(new ZipWriter());
+ providers.add(new TikaServerParseExceptionMapper());
+ sf.setProviders(providers);
+ }
+
+ @Override
+ protected InputStream getPipesConfigInputStream() throws IOException {
+ unpackTempDir = Files.createTempDirectory("tika-unpack-fd-");
+ Map<String, Object> replacements = new HashMap<>();
+ replacements.put("UNPACK_EMITTER_BASE_PATH",
unpackTempDir.toAbsolutePath().toString());
+ replacements.put("PLUGINS_PATHS",
+
Paths.get("target/plugins").toAbsolutePath().toString().replace("\\", "/"));
+ replacements.put("TIMEOUT_MILLIS", 60000L);
+ JsonNode config = JsonConfigHelper.loadFromResource(
+ "/configs/cxf-unpack-test-template.json", CXFTestBase.class,
replacements);
+ return new ByteArrayInputStream(
+
MAPPER.writeValueAsString(config).getBytes(StandardCharsets.UTF_8));
+ }
+
+ @Override
+ protected Path getUnpackEmitterBasePath() {
+ return unpackTempDir;
+ }
+
+ @Test
+ public void testServerConfigMakesUnpackFrictionless() throws Exception {
+ Map<String, byte[]> entries = unpack("/unpack");
+ assertTrue(entries.containsKey("datapackage.json"),
entries.keySet().toString());
+ assertTrue(entries.keySet().stream().anyMatch(k ->
k.startsWith("unpacked/")),
+ entries.keySet().toString());
+ }
+
+ /** A Frictionless package carries metadata.json by default; /all adds
only the original. */
+ @Test
+ public void testPlainUnpackHasMetadataJsonByDefault() throws Exception {
+ Map<String, byte[]> entries = unpack("/unpack");
+ assertTrue(entries.containsKey("metadata.json"),
entries.keySet().toString());
+ assertFalse(entries.keySet().stream().anyMatch(k ->
k.startsWith("unpacked/0.")),
+ "plain /unpack must not pack the container's bytes: " +
entries.keySet());
+ }
+
+ /** DIRECTORY from a request contradicts the one-body transport: 400, not
a silent pin. */
+ @Test
+ public void testDirectoryModeInRequestIs400() throws Exception {
+ ContentDisposition cd = new ContentDisposition(
+ "form-data; name=\"file\";
filename=\"test_recursive_embedded.docx\"");
+ Attachment fileAtt = new Attachment("file",
ClassLoader.getSystemResourceAsStream(TEST_DOC), cd);
+ Attachment configAtt = new Attachment("config", "application/json",
new ByteArrayInputStream(
+ "{\"unpack-config\": {\"outputMode\":
\"DIRECTORY\"}}".getBytes(StandardCharsets.UTF_8)));
+ Response response = WebClient.create(endPoint + "/unpack")
+ .type("multipart/form-data")
+ .accept("application/zip")
+ .post(new MultipartBody(Arrays.asList(fileAtt, configAtt)));
+ assertEquals(400, response.getStatus());
+ }
+
+ @Test
+ public void testUnpackAllCarriesMetadataAndTheOriginal() throws Exception {
+ Map<String, byte[]> entries = unpack("/unpack/all");
+ assertTrue(entries.containsKey("metadata.json"),
entries.keySet().toString());
+ assertTrue(entries.keySet().stream().anyMatch(k ->
k.startsWith("unpacked/0.")),
+ "includeOriginal should add the container: " +
entries.keySet());
+ }
+
+ /** The RMETA list's row 0 is the container -- its own metadata and its
own text. */
+ @Test
+ public void testMetadataJsonCarriesTheContainerRow() throws Exception {
+ Map<String, byte[]> entries = unpack("/unpack");
+ JsonNode rows = MAPPER.readTree(entries.get("metadata.json"));
+ assertTrue(rows.isArray() && rows.size() > 1, "expected an RMETA list,
got: " + rows);
+
+ JsonNode container = rows.get(0);
+ assertEquals("0", container.path("tk:embedded-depth").asText(),
+ "row 0 should be the container: " + container);
+ assertTrue(container.hasNonNull("tk:content"),
+ "the container row should carry its own extracted text: " +
container);
+ assertTrue(container.hasNonNull("dcterms:created"),
+ "the container row should carry its own metadata: " +
container);
+ }
+
+ /** Every row can be joined to its file: id, tree position and logical
path all travel. */
+ @Test
+ public void testRowsCarryEmbeddedPathData() throws Exception {
+ Map<String, byte[]> entries = unpack("/unpack/all");
+ JsonNode rows = MAPPER.readTree(entries.get("metadata.json"));
+ JsonNode embedded = rows.get(1);
+ for (String key : new String[]{"tk:embedded-id", "tk:embedded-id-path",
+ "tk:embedded-resource-path", "tk:resource-name"}) {
+ assertTrue(embedded.hasNonNull(key), key + " missing from: " +
embedded);
+ }
+ String id = embedded.get("tk:embedded-id").asText();
+ assertTrue(entries.keySet().stream().anyMatch(k ->
k.startsWith("unpacked/" + id + ".")),
+ "row's tk:embedded-id should name a packed file: " +
entries.keySet());
+ }
+
+ private Map<String, byte[]> unpack(String path) throws Exception {
+ Response response = WebClient.create(endPoint + path)
+ .accept("application/zip")
+ .put(ClassLoader.getSystemResourceAsStream(TEST_DOC));
+ assertEquals(200, response.getStatus());
+ return readZipArchiveBytes((InputStream) response.getEntity());
+ }
+}
diff --git
a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackerResourceHandlerTest.java
b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackerResourceHandlerTest.java
new file mode 100644
index 0000000000..6b4842262e
--- /dev/null
+++
b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackerResourceHandlerTest.java
@@ -0,0 +1,267 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.server.standard;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import jakarta.ws.rs.core.Response;
+import org.apache.cxf.jaxrs.JAXRSServerFactoryBean;
+import org.apache.cxf.jaxrs.client.WebClient;
+import org.apache.cxf.jaxrs.ext.multipart.Attachment;
+import org.apache.cxf.jaxrs.ext.multipart.ContentDisposition;
+import org.apache.cxf.jaxrs.ext.multipart.MultipartBody;
+import org.apache.cxf.jaxrs.lifecycle.SingletonResourceProvider;
+import org.junit.jupiter.api.Test;
+
+import org.apache.tika.serialization.config.JsonConfigHelper;
+import org.apache.tika.server.core.CXFTestBase;
+import org.apache.tika.server.core.TikaServerParseExceptionMapper;
+import org.apache.tika.server.core.resource.UnpackerResource;
+import org.apache.tika.server.core.writer.TarWriter;
+import org.apache.tika.server.core.writer.ZipWriter;
+
+/**
+ * /unpack has no format segment, so the config part is the only place a
handler is named --
+ * and it reaches the RMETA list written to metadata.json.
+ */
+public class UnpackerResourceHandlerTest extends CXFTestBase {
+
+ private static final String TEST_DOC =
"test-documents/test_recursive_embedded.docx";
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+ private static final String FRICTIONLESS =
+ "\"unpack-config\": {\"outputFormat\": \"FRICTIONLESS\",
\"outputMode\": \"ZIPPED\", "
+ + "\"includeFullMetadata\": true}";
+
+ private Path unpackTempDir;
+
+ @Override
+ protected boolean isAllowPerRequestConfig() {
+ return true;
+ }
+
+ @Override
+ protected void setUpResources(JAXRSServerFactoryBean sf) {
+ sf.setResourceClasses(UnpackerResource.class);
+ sf.setResourceProvider(UnpackerResource.class,
+ new SingletonResourceProvider(new
UnpackerResource(tikaResource)));
+ }
+
+ @Override
+ protected void setUpProviders(JAXRSServerFactoryBean sf) {
+ List<Object> providers = new ArrayList<>();
+ providers.add(new TarWriter());
+ providers.add(new ZipWriter());
+ providers.add(new TikaServerParseExceptionMapper());
+ sf.setProviders(providers);
+ }
+
+ @Override
+ protected InputStream getTikaConfigInputStream() throws IOException {
+ return
this.getClass().getResourceAsStream("/configs/tika-config-unpacker.json");
+ }
+
+ @Override
+ protected InputStream getPipesConfigInputStream() throws IOException {
+ unpackTempDir = Files.createTempDirectory("tika-unpack-probe-");
+ Map<String, Object> replacements = new HashMap<>();
+ replacements.put("UNPACK_EMITTER_BASE_PATH",
unpackTempDir.toAbsolutePath().toString());
+ replacements.put("PLUGINS_PATHS",
+
Paths.get("target/plugins").toAbsolutePath().toString().replace("\\", "/"));
+ replacements.put("TIMEOUT_MILLIS", 60000L);
+ JsonNode config = JsonConfigHelper.loadFromResource(
+ "/configs/cxf-unpack-test-template.json", CXFTestBase.class,
replacements);
+ return new ByteArrayInputStream(
+
MAPPER.writeValueAsString(config).getBytes(StandardCharsets.UTF_8));
+ }
+
+ @Override
+ protected Path getUnpackEmitterBasePath() {
+ return unpackTempDir;
+ }
+
+ @Test
+ public void testMetadataJsonDefaultsToMarkdown() throws Exception {
+ String json = metadataJson("{" + FRICTIONLESS + "}");
+ assertFalse(json.contains("<html xmlns="), "default handler should not
be XHTML");
+ }
+
+ @Test
+ public void testConfigPartSelectsTheMetadataJsonHandler() throws Exception
{
+ String json = metadataJson("{" + FRICTIONLESS
+ + ", \"basic-content-handler-factory\": {\"type\": \"XML\"}}");
+ assertTrue(json.contains("<html xmlns="),
+ "config part should select the handler for metadata.json");
+ }
+
+ /** /unpack/all/<handler> used to 200 and silently ignore the segment. */
+ @Test
+ public void testHandlerSegmentIsHonoredNotSwallowed() throws Exception {
+ assertEquals(0, sidecarsWithContent(putAll("/unpack/all/ignore")),
+ "ignore in the path must suppress tk:content");
+ assertTrue(sidecarsWithContent(putAll("/unpack/all")) > 0, "baseline
carries tk:content");
+ }
+
+ /** Plain /unpack carries no metadata for a handler to render, so it takes
no segment. */
+ @Test
+ public void testNoHandlerSegmentOnPlainUnpack() throws Exception {
+ assertEquals(404, WebClient.create(endPoint + "/unpack/xml")
+ .accept("application/zip")
+
.put(ClassLoader.getSystemResourceAsStream(TEST_DOC)).getStatus());
+ }
+
+ /** The 4.0 config-variant spelling still works, and takes the handler
after it. */
+ @Test
+ public void testAllConfigSpellingIsNotAHandlerNamedConfig() throws
Exception {
+ Response response = post("/unpack/all/config", "{" + FRICTIONLESS +
"}");
+ assertEquals(200, response.getStatus());
+ assertNotNull(readZipArchiveBytes((InputStream)
response.getEntity()).get("metadata.json"));
+ response = post("/unpack/all/config/ignore", "{" + FRICTIONLESS + "}");
+ assertEquals(200, response.getStatus());
+ String json = new String(readZipArchiveBytes((InputStream)
response.getEntity())
+ .get("metadata.json"), StandardCharsets.UTF_8);
+ assertFalse(json.contains("\"tk:content\""), "ignore after /config
must suppress tk:content");
+ }
+
+ @Test
+ public void testUnrecognizedHandlerSegmentIsBadRequest() throws Exception {
+ assertEquals(400, WebClient.create(endPoint +
"/unpack/all/somethingOrOther")
+ .accept("application/zip")
+
.put(ClassLoader.getSystemResourceAsStream(TEST_DOC)).getStatus());
+ }
+
+ private List<byte[]> putAll(String path) throws Exception {
+ Response response = WebClient.create(endPoint + path)
+ .accept("application/zip")
+ .put(ClassLoader.getSystemResourceAsStream(TEST_DOC));
+ assertEquals(200, response.getStatus());
+ return sidecarsOf(response);
+ }
+
+ /** includeMetadataInZip writes a sidecar per extracted file; it carries
tk:content too. */
+ @Test
+ public void testPerFileSidecarsFollowTheConfigPartHandler() throws
Exception {
+ assertTrue(perFileSidecar("{\"unpack-config\":
{\"includeMetadataInZip\": true}}")
+ .contains("\"tk:content-handler-type\":\"MARKDOWN\""));
+
+ String xml = perFileSidecar("{\"unpack-config\":
{\"includeMetadataInZip\": true}, "
+ + "\"basic-content-handler-factory\": {\"type\": \"XML\"}}");
+ assertTrue(xml.contains("\"tk:content-handler-type\":\"XML\""), xml);
+ assertTrue(xml.contains("<html xmlns="), xml);
+ }
+
+ /** IGNORE is the opt-out: sidecars keep their metadata but carry no
extracted text. */
+ @Test
+ public void testIgnoreHandlerLeavesSidecarsWithoutContent() throws
Exception {
+ String withText = "{\"unpack-config\": {\"includeMetadataInZip\":
true}}";
+ String ignore = "{\"unpack-config\": {\"includeMetadataInZip\": true},
"
+ + "\"basic-content-handler-factory\": {\"type\": \"IGNORE\"}}";
+
+ assertTrue(sidecarsWithContent(withText) > 0, "baseline should carry
tk:content");
+ assertEquals(0, sidecarsWithContent(ignore), "IGNORE should suppress
tk:content");
+ assertTrue(sidecarCount(ignore) > 0, "IGNORE must not drop the
sidecars themselves");
+ }
+
+ private int sidecarsWithContent(String configJson) throws Exception {
+ return sidecarsWithContent(sidecars(configJson));
+ }
+
+ private int sidecarsWithContent(List<byte[]> sidecars) throws Exception {
+ int count = 0;
+ for (byte[] sidecar : sidecars) {
+ if (new String(sidecar,
StandardCharsets.UTF_8).contains("\"tk:content\"")) {
+ count++;
+ }
+ }
+ return count;
+ }
+
+ private int sidecarCount(String configJson) throws Exception {
+ return sidecars(configJson).size();
+ }
+
+ private List<byte[]> sidecars(String configJson) throws Exception {
+ Response response = post("/unpack", configJson);
+ assertEquals(200, response.getStatus());
+ return sidecarsOf(response);
+ }
+
+ private List<byte[]> sidecarsOf(Response response) throws Exception {
+ Map<String, byte[]> entries = readZipArchiveBytes((InputStream)
response.getEntity());
+ List<byte[]> out = new ArrayList<>();
+ for (Map.Entry<String, byte[]> e : entries.entrySet()) {
+ if (e.getKey().endsWith(".metadata.json")) {
+ out.add(e.getValue());
+ }
+ }
+ return out;
+ }
+
+ private String perFileSidecar(String configJson) throws Exception {
+ Response response = post("/unpack", configJson);
+ assertEquals(200, response.getStatus());
+ Map<String, byte[]> entries = readZipArchiveBytes((InputStream)
response.getEntity());
+ for (Map.Entry<String, byte[]> e : entries.entrySet()) {
+ if (e.getKey().endsWith(".metadata.json")) {
+ String json = new String(e.getValue(), StandardCharsets.UTF_8);
+ if (json.contains("\"tk:content\"")) {
+ return json;
+ }
+ }
+ }
+ throw new AssertionError("no per-file sidecar carrying tk:content");
+ }
+
+ private String metadataJson(String configJson) throws Exception {
+ Response response = post("/unpack", configJson);
+ assertEquals(200, response.getStatus());
+ Map<String, byte[]> entries = readZipArchiveBytes((InputStream)
response.getEntity());
+ byte[] md = entries.get("metadata.json");
+ assertNotNull(md, "includeFullMetadata should write metadata.json");
+ return new String(md, StandardCharsets.UTF_8);
+ }
+
+ private Response post(String path, String configJson) {
+ ContentDisposition cd = new ContentDisposition(
+ "form-data; name=\"file\";
filename=\"test_recursive_embedded.docx\"");
+ Attachment fileAtt = new Attachment("file",
+ ClassLoader.getSystemResourceAsStream(TEST_DOC), cd);
+ Attachment configAtt = new Attachment("config", "application/json",
+ new
ByteArrayInputStream(configJson.getBytes(StandardCharsets.UTF_8)));
+ return WebClient.create(endPoint + path)
+ .type("multipart/form-data")
+ .accept("application/zip")
+ .post(new MultipartBody(Arrays.asList(fileAtt, configAtt)));
+ }
+}
diff --git
a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackerResourceTest.java
b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackerResourceTest.java
index 1ae4bb0592..274c0cb24f 100644
---
a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackerResourceTest.java
+++
b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackerResourceTest.java
@@ -661,8 +661,8 @@ public class UnpackerResourceTest extends CXFTestBase {
/**
* Documents the shape of /unpack output in FRICTIONLESS mode (no /all,
- * default config): a datapackage.json manifest plus the
unpacked/<id>
- * children, and no metadata.json envelope.
+ * default config): a datapackage.json manifest, the metadata.json envelope
+ * a Frictionless package carries by default, and the unpacked/<id>
children.
*/
@Test
public void testFrictionlessUnpackShape() throws Exception {
@@ -693,8 +693,8 @@ public class UnpackerResourceTest extends CXFTestBase {
assertTrue(data.containsKey("datapackage.json"),
"Should contain datapackage.json manifest. Entries: " +
data.keySet());
- assertFalse(data.containsKey("metadata.json"),
- "Should not contain metadata.json without includeFullMetadata.
Entries: " + data.keySet());
+ assertTrue(data.containsKey("metadata.json"),
+ "A Frictionless package carries metadata.json by default.
Entries: " + data.keySet());
boolean hasUnpacked = data.keySet().stream().anyMatch(k ->
k.startsWith("unpacked/"));
assertTrue(hasUnpacked, "Should contain unpacked/ entries. Entries: "
+ data.keySet());
}
