This is an automated email from the ASF dual-hosted git repository. tballison pushed a commit to branch TIKA-4971 in repository https://gitbox.apache.org/repos/asf/tika.git
commit 8fe5cf2084419e65b9dc2ac7148188a58c1e5bc7 Author: tballison <[email protected]> AuthorDate: Tue Oct 13 21:54:31 2026 +0100 TIKA-4971: make server and apps unpack output the same and make frictionless output validate --- .skills/users/file-forensics/SKILL.md | 22 ++- .skills/users/file-to-markdown/SKILL.md | 12 +- CHANGES.txt | 14 ++ .../advanced/integration-testing/tika-app.adoc | 2 +- .../advanced/integration-testing/tika-server.adoc | 2 +- docs/modules/ROOT/pages/pipes/unpack-config.adoc | 87 +++++++++--- docs/modules/ROOT/pages/using-tika/cli/index.adoc | 11 +- .../ROOT/pages/using-tika/server/index.adoc | 4 +- .../src/main/java/org/apache/tika/cli/TikaCLI.java | 2 +- .../test/java/org/apache/tika/cli/TikaCLITest.java | 155 +++++++++++++++------ .../org/apache/tika/async/cli/TikaAsyncCLI.java | 24 +++- .../core/extractor/FrictionlessUnpackHandler.java | 8 +- .../tika/pipes/core/extractor/UnpackConfig.java | 69 ++++++++- .../core/extractor/frictionless/DataPackage.java | 20 +-- .../frictionless/FrictionlessResource.java | 57 ++++---- .../apache/tika/pipes/core/server/EmitHandler.java | 48 +++++-- .../core/server/MetadataListAndEmbeddedBytes.java | 13 ++ .../apache/tika/pipes/core/server/PipesWorker.java | 130 ++++++++++------- .../apache/tika/pipes/core/presets/thumbnail.json | 2 + .../frictionless/FrictionlessResourceTest.java | 40 ++++++ .../tika/pipes/core/FrictionlessUnpackTest.java | 83 ++++++++--- .../org/apache/tika/pipes/core/UnpackModeTest.java | 17 ++- .../server/core/resource/PipesParsingHelper.java | 3 + .../tika/server/core/resource/TikaResource.java | 18 +++ .../server/core/resource/UnpackerResource.java | 12 +- .../apache/tika/server/core/StackTraceTest.java | 10 +- .../tika/server/standard/ThumbnailPresetTest.java | 2 +- .../apache/tika/server/standard/TikaPipesTest.java | 2 +- .../server/standard/UnpackFrictionlessTest.java | 12 +- .../server/standard/UnpackRegularConfigTest.java | 2 +- .../standard/UnpackerResourceHandlerTest.java | 14 +- .../tika/server/standard/UnpackerResourceTest.java | 76 +++++++--- 32 files changed, 697 insertions(+), 276 deletions(-) diff --git a/.skills/users/file-forensics/SKILL.md b/.skills/users/file-forensics/SKILL.md index 0675d28b5d..b774d28dee 100644 --- a/.skills/users/file-forensics/SKILL.md +++ b/.skills/users/file-forensics/SKILL.md @@ -106,7 +106,7 @@ java -jar tika-app.jar --config=file-forensics-config.json -Z --extract-dir=evid # mkdir -p evidence/suspect-embedded && unzip -q suspect-embedded.zip -d evidence/suspect-embedded jq 'map(del(."tk:content"))' suspect.rmeta.json > suspect.meta.json -ls -lR evidence/suspect-embedded && sha256sum evidence/suspect-embedded/*/unpacked/* 2>/dev/null || sha256sum evidence/suspect-embedded/unpacked/* +ls -lR evidence/suspect-embedded && sha256sum evidence/suspect-embedded/unpacked/* ``` `suspect.rmeta.json` is the full evidence record — a JSON array where entry @@ -339,20 +339,18 @@ files is incomplete. back visually corrupted (`VB\_Name`). The `-z`/`-Z` extracted files are pristine bytes. (Alternative: capture with `-J -t` for unescaped plain text per entry.) -- Extracted files are **renumbered** (`00000001.jpg`, ...); original names - live in each rmeta entry's `tk:resource-name`, and `-z`/`-Z` writes a - sidecar `<name>.json` metadata dump for mapping numbers back to names. -- The output is a Frictionless data package (Tika 4.2+): the files sit under - `<container>/unpacked/`, and `<container>/datapackage.json` already carries a - SHA-256 per file. Digest what you extracted - (`sha256sum evidence/suspect-embedded/*/unpacked/*`) and check it against the +- Extracted files are **renumbered** (`1.jpg`, `2.pdf`, ...); each + `datapackage.json` resource carries the original name (`tk:resource-name`) + and `tk:embedded-id`, the key to the file's row in `rmeta.json`. +- The output is a Frictionless data package (Tika 4.2+): for a single input + file the extract dir is the package — files under `unpacked/`, and + `datapackage.json` already carries a SHA-256 per file (and for + `rmeta.json`). Digest what you extracted + (`sha256sum evidence/suspect-embedded/unpacked/*`) and check it against the manifest, so each artifact is pinned the same way the container file is. - Mechanics and gotchas (Pipes-mode delay, chatty stderr) are in `file-to-markdown`; the server-side equivalent is `/unpack`, which returns - the same package as one zip over HTTP. **`/unpack` names differ from - `-z`/`-Z`:** plain sequential names (`unpacked/1.jpg`, `unpacked/2.pdf`, ...) - and the rmeta rows live in the package's `metadata.json` rather than a - sidecar — map names back via each row's `tk:resource-name`. Against the + the same package as one zip over HTTP, laid out the same way. Against the server started above (forensics config loaded): ```bash diff --git a/.skills/users/file-to-markdown/SKILL.md b/.skills/users/file-to-markdown/SKILL.md index 51c410007b..0fb0d8b4fa 100644 --- a/.skills/users/file-to-markdown/SKILL.md +++ b/.skills/users/file-to-markdown/SKILL.md @@ -151,11 +151,13 @@ embedded content in: email attachments (`.eml`, `.msg`), archive members (`.zip`), PDF attachments, embedded objects/images in any office format (an inline pasted picture counts), and so on. -What lands on disk: a `<name>/` Frictionless package — `datapackage.json` -(SHA-256 per file), `metadata.json`, and the embedded files **renumbered** -under `unpacked/` (`00000001.jpg`, ...). Original names are not preserved on -disk; they are in each rmeta entry's `tk:resource-name`, so keep the sibling -`<name>.json` (an rmeta-shaped dump `-z` also writes) to map numbers back. +What lands on disk: the extract dir (default `<name>-unpacked/`) is a +Frictionless package — `datapackage.json` (SHA-256 per file), +`rmeta.json` (the rmeta list), and the embedded files **renumbered** +under `unpacked/` (`1.jpg`, ...). Original names are not preserved on disk; +each `datapackage.json` resource has `tk:resource-name` and `tk:embedded-id` +(its row in `rmeta.json`). An input +directory gets one such package per file, under its relative path. `-z`/`-Z` route through Tika Pipes mode rather than the fast synchronous path the other flags use — expect several seconds and a burst of diff --git a/CHANGES.txt b/CHANGES.txt index 48dc600091..2aaf28d427 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,5 +1,19 @@ Release 4.2.0 - unreleased + * Frictionless unpack: the package's metadata.json is renamed rmeta.json, is listed in + datapackage.json as a resource (size, sha256), and now gets the metadata filters and + user metadata the emitted list gets; resources are in embedded-id order. A document + with no attachments is a package of datapackage.json and rmeta.json: tika-server + /unpack returns it with 200 instead of 204 (unpack-config.emptyPackage=SKIP, which the + thumbnail preset sets, restores the 204). The package name is the caller's filename + on tika-server, never the server's spool file. tika-app -z/-Z on a single file makes + the output dir the package (default <name>-unpacked/), names files unpacked/1.<ext> + as tika-server /unpack does, and no longer writes a <name>.json next to it. New + unpack-config.packageRoot and packageName set where a DIRECTORY-mode package goes + and its name. Packages pass frictionless validate: each resource's name is derived + from its path, the original filename moves to tk:resource-name, and tk:embedded-id + keys it to its rmeta.json row (TIKA-4971). + * Config properties whose value reaches the host (paths, executables, class names, URLs, credentials, operator-sized limits) are marked @OperatorOnly and refused in per-request JSON before any setter runs. Newly locked: pdf-parser's diff --git a/docs/modules/ROOT/pages/advanced/integration-testing/tika-app.adoc b/docs/modules/ROOT/pages/advanced/integration-testing/tika-app.adoc index 08b41395c1..3dcb754f42 100644 --- a/docs/modules/ROOT/pages/advanced/integration-testing/tika-app.adoc +++ b/docs/modules/ROOT/pages/advanced/integration-testing/tika-app.adoc @@ -338,7 +338,7 @@ ls -R /tmp/tika-app-test/frictionless-out ---- *Expected:* `test_recursive_embedded.docx.json` plus a `test_recursive_embedded.docx/` -directory holding `datapackage.json`, `metadata.json` and `unpacked/`. +directory holding `datapackage.json`, `rmeta.json` and `unpacked/`. === Test 24: Unpack zipped, and the pre-4.2 flat layout diff --git a/docs/modules/ROOT/pages/advanced/integration-testing/tika-server.adoc b/docs/modules/ROOT/pages/advanced/integration-testing/tika-server.adoc index 414bb770f1..da378cfa6c 100644 --- a/docs/modules/ROOT/pages/advanced/integration-testing/tika-server.adoc +++ b/docs/modules/ROOT/pages/advanced/integration-testing/tika-server.adoc @@ -158,7 +158,7 @@ curl -s -X PUT -T test_recursive_embedded.docx http://localhost:9998/unpack/all unzip -l /tmp/unpack.zip ---- -*Expected:* A Frictionless package: `datapackage.json`, `metadata.json`, and the embedded +*Expected:* A Frictionless package: `datapackage.json`, `rmeta.json`, and the embedded files plus the original container (`0.docx`) under `unpacked/`. === Test 13: GET /parsers diff --git a/docs/modules/ROOT/pages/pipes/unpack-config.adoc b/docs/modules/ROOT/pages/pipes/unpack-config.adoc index 9ac0881ca7..36a579cf92 100644 --- a/docs/modules/ROOT/pages/pipes/unpack-config.adoc +++ b/docs/modules/ROOT/pages/pipes/unpack-config.adoc @@ -103,7 +103,7 @@ immediately capped. |`includeMetadata` |boolean |unset: on for `FRICTIONLESS`, off for `REGULAR` -|Metadata for every extracted file, in either format: one `metadata.json` under Frictionless (a data package without the parse is half a package, so it is on unless you set `false`), a `.metadata.json` sidecar per file under REGULAR zip output (opt-in; tika-server's `/unpack/all` sets it). REGULAR loose output (no zip) writes none. +|Metadata for every extracted file, in either format: one `rmeta.json` under Frictionless (a data package without the parse is half a package, so it is on unless you set `false`), a `.metadata.json` sidecar per file under REGULAR zip output (opt-in; tika-server's `/unpack/all` sets it). REGULAR loose output (no zip) writes none. |`includeMetadataInZip` |boolean @@ -145,6 +145,21 @@ immediately capped. |`ZIPPED` |`ZIPPED` packages everything into one archive; `DIRECTORY` emits each extracted file to the emitter as its own item. +|`packageRoot` +|String +|_(container's emit key)_ +|`FRICTIONLESS` + `DIRECTORY` only: emit key prefix of the package. Unset, each container gets `<emitKey>/datapackage.json`, `<emitKey>/unpacked/...`; `""` writes the package at the emitter's root (tika-app does this for a single input file). + +|`emptyPackage` +|WRITE, SKIP +|`WRITE` +|`FRICTIONLESS` only: a document with nothing to unpack still gets a package (`datapackage.json` and `rmeta.json`), or with `SKIP` none -- and tika-server's `/unpack` answers `204`. The `thumbnail` preset sets `SKIP`. + +|`packageName` +|String +|_(fetch key's file name)_ +|`FRICTIONLESS` only: the `name` in `datapackage.json`; blank leaves it out. tika-server sets the caller's filename, or blank when there is none. + |`includeFullMetadata` |boolean |-- @@ -272,14 +287,14 @@ A zipped package contains: ---- output.zip ├── datapackage.json # Manifest with file list, SHA256 hashes, mimetypes -├── metadata.json # Full RMETA metadata (unless includeMetadata=false) +├── rmeta.json # Full RMETA metadata (unless includeMetadata=false) └── unpacked/ - ├── 00000001.pdf - ├── 00000002.png + ├── 1.pdf + ├── 2.png └── ... ---- -The `tk:content` field inside `metadata.json` (and inside the per-file `.metadata.json` +The `tk:content` field inside `rmeta.json` (and inside the per-file `.metadata.json` sidecars `includeMetadata` adds to REGULAR zip output) is Markdown by default. On tika-server change it with `/unpack/all/{handlerType}` or a `content-handler-factory` in the `config` part; in a config, the factory alone. Each entry records the handler it was written with in @@ -321,9 +336,16 @@ Naming the handler in both the path and a `config` part is a `400`, as on `/tika == What each surface produces The five knobs interact differently per surface. Two defaults changed since 4.0.0: a -Frictionless package carries `metadata.json` unless `includeMetadata` is `false` (4.0.0 left +Frictionless package carries `rmeta.json` unless `includeMetadata` is `false` (4.0.0 left it out unless `includeFullMetadata` was `true`), and since 4.2 Frictionless is the format -on every surface. REGULAR output is as it was. +on every surface. REGULAR output is as it was. Before 4.2 the package's `rmeta.json` was +named `metadata.json`, was not listed in `datapackage.json`, and did not get the metadata +filters. + +A document with nothing to unpack is still a package: `datapackage.json` and `rmeta.json`, +with `rmeta.json` its only resource and no `unpacked/`. `/unpack` returns it with `200` +rather than the `204` it gave before 4.2 (REGULAR output still gives `204`). To get the `204` +back, send `"unpack-config": {"emptyPackage": "SKIP"}` in the request's `config` part. [cols="1,2,2,2"] |=== @@ -361,7 +383,7 @@ on every surface. REGULAR output is as it was. |=== `/unpack/all` means the source document's bytes plus metadata in both formats. Under REGULAR -that is where the per-file sidecars come from; under Frictionless `metadata.json` is already +that is where the per-file sidecars come from; under Frictionless `rmeta.json` is already there, so `/all` adds only the source document. `outputMode: DIRECTORY` is refused on `/unpack`: an HTTP response is one zip, so a request asking for it gets a `400`, and a server whose own `unpack-config` says so fails at startup if the `unpack` endpoint is enabled -- use @@ -371,11 +393,15 @@ Output trees for `doc.docx` with three attachments: [source] ---- -tika-server (one zip) tika-app -Z (loose) - datapackage.json out/doc.docx.json - metadata.json out/doc.docx/datapackage.json - unpacked/1.emf ... out/doc.docx/metadata.json - + with /all: unpacked/0.docx out/doc.docx/unpacked/1.emf ... +tika-server (one zip) tika-app -Z doc.docx out (loose) + datapackage.json out/datapackage.json + rmeta.json out/rmeta.json + unpacked/1.emf ... out/unpacked/1.emf ... + + with /all: unpacked/0.docx + tika-app -Z <dir> out (loose) + out/doc.docx/datapackage.json + out/doc.docx/rmeta.json + out/doc.docx/unpacked/1.emf ... REGULAR, tika-server (one zip) REGULAR, tika-app -Z --unpack-format=REGULAR 1.emf out/doc.docx.json @@ -385,16 +411,43 @@ REGULAR, tika-server (one zip) REGULAR, tika-app -Z --unpack-format=REGUL <name>.metadata.json per file ---- -The top-level `doc.docx.json` on the command line is the normal pipes output for the parse; -the package's `metadata.json` carries the same rows again so it is self-contained. Set -`includeMetadata: false` in a `-c` file to drop it. +For a single input file the output directory is the package; with no output directory +tika-app writes it to `doc.docx-unpacked/`. For a directory of inputs each gets its own +package under its relative path. The package's `rmeta.json` is the parse's metadata list, +so no separate `doc.docx.json` is written; with `includeMetadata: false` (in a `-c` file) +the list goes back to `doc.docx.json` next to the package. The `datapackage.json` file contains: -* List of all extracted files as "resources" +* List of all extracted files and `rmeta.json` as "resources" * SHA256 hash for each file * MIME type for each file * File size in bytes +* `name`: the Frictionless resource name, derived from the path (`unpacked/3.pdf` -> + `unpacked-3.pdf`) +* `tk:embedded-id`: for an extracted file, the `tk:embedded-id` of its row in `rmeta.json` + (`0` is the source document, under `includeOriginal`). The rows are not in id order; match + on the id. +* `tk:resource-name`: the file's original name, when it had one + +The package passes `frictionless validate`: + +[source,json] +---- +{ + "name" : "doc.docx", + "resources" : [ + { "name" : "rmeta.json", "path" : "rmeta.json", "mediatype" : "application/json", ... }, + { "name" : "unpacked-3.pdf", "path" : "unpacked/3.pdf", "mediatype" : "application/pdf", + "bytes" : 109786, "hash" : "sha256:115d...", "tk:embedded-id" : 3, + "tk:resource-name" : "report.pdf" } + ] +} +---- + +One thing the validator reports that is not a manifest problem: it decompresses resources +whose extension says `.zip` or `.gz`, so a corrupt or truncated embedded archive shows up +as a `compression-error`. The manifest is still valid; the file is what it is. === CLI Usage diff --git a/docs/modules/ROOT/pages/using-tika/cli/index.adoc b/docs/modules/ROOT/pages/using-tika/cli/index.adoc index 1835b30749..d70e759be7 100644 --- a/docs/modules/ROOT/pages/using-tika/cli/index.adoc +++ b/docs/modules/ROOT/pages/using-tika/cli/index.adoc @@ -399,19 +399,20 @@ through every level. Both accept a single file or a directory as input. |Output format: `FRICTIONLESS` (default) or `REGULAR`, the pre-4.2 flat layout. |`--unpack-mode=<mode>` -|Output mode: `DIRECTORY` (default) or `ZIPPED`. `DIRECTORY` lays the package out under -`<out>/<input>/` (`datapackage.json`, `metadata.json`, `unpacked/...`); `ZIPPED` writes -`<out>/<input>-frictionless.zip`. +|Output mode: `DIRECTORY` (default) or `ZIPPED`. `DIRECTORY` with a single input file makes +`<out>` the package (`datapackage.json`, `rmeta.json`, `unpacked/...`; `<out>` defaults +to `<input>-unpacked`); with an input directory each file gets `<out>/<input>/`. `ZIPPED` +writes `<out>/<input>-frictionless.zip`. |`--unpack-include-metadata` -|Metadata for every extracted file. A Frictionless package carries `metadata.json` by default; +|Metadata for every extracted file. A Frictionless package carries `rmeta.json` by default; this adds a `.metadata.json` sidecar per file to REGULAR zip output. |=== A `-c`/`--config` file's `unpack-config` is honoured and the flags above are applied on top of it. That is the only way to get the source document itself into the output (`"unpack-config": {"includeOriginal": true}`); no flag does it. File naming stays the -CLI's own (`<input>/unpacked/00000001.<ext>`, or `<input>-embed/00000001.<ext>` under +CLI's own (`unpacked/1.<ext>`, or `<input>-embed/1.<ext>` under `REGULAR`, detected suffixes, loose files unless `--unpack-mode=ZIPPED`) regardless of the file's `suffixStrategy`, `zeroPadName`, `embeddedIdPrefix`, `keyBaseStrategy` or `outputMode`. See xref:pipes/unpack-config.adoc#_what_each_surface_produces[what each surface produces]. diff --git a/docs/modules/ROOT/pages/using-tika/server/index.adoc b/docs/modules/ROOT/pages/using-tika/server/index.adoc index bc27b79b17..3b822840a5 100644 --- a/docs/modules/ROOT/pages/using-tika/server/index.adoc +++ b/docs/modules/ROOT/pages/using-tika/server/index.adoc @@ -212,7 +212,7 @@ config's, then the endpoint default (Markdown). Output format: name the handler once. A format segment sets it -- `/tika/config/xml`, `/rmeta/config/xml`, and `/unpack/all/{handlerType}` (or `/unpack/all/config/{handlerType}` with a config part), which selects how `tk:content` is rendered in the metadata `/all` adds -(`/unpack/all/ignore` for none). Plain `/unpack` takes no segment: the `metadata.json` a +(`/unpack/all/ignore` for none). Plain `/unpack` takes no segment: the `rmeta.json` a Frictionless package carries there uses the config's handler, or the default. With no segment, a `content-handler-factory` in the `config` part does. Doing both is a `400`, whether or not the @@ -577,7 +577,7 @@ before the exception are discarded. Known gap. By default exception detail is returned in full. Stack traces and their messages can contain the spooled file's path, the source filename, and fragments of the document. That detail appears in `tk:exception:*` metadata on **successful** parses as well as in error bodies — including inside -the `metadata.json` of an `/unpack` zip. +the `rmeta.json` of an `/unpack` zip. One config setting governs every one of those channels — the metadata values, every error body built from an exception (`422`, `500`; a `400` echoes the caller's own request problem, `415` and diff --git a/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java b/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java index f52cdfab27..00213e5423 100644 --- a/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java +++ b/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java @@ -998,7 +998,7 @@ public class TikaCLI { out.println(" --unpack-format=<format> Output format: FRICTIONLESS (default) or REGULAR"); out.println(" --unpack-mode=<mode> Output mode: DIRECTORY (default: loose files) or ZIPPED"); out.println(" --unpack-include-metadata Metadata for every extracted file. A Frictionless package"); - out.println(" carries metadata.json by default; this adds per-file"); + out.println(" carries rmeta.json by default; this adds per-file"); out.println(" sidecars to REGULAR zip output"); out.println(); out.println(); diff --git a/tika-app/src/test/java/org/apache/tika/cli/TikaCLITest.java b/tika-app/src/test/java/org/apache/tika/cli/TikaCLITest.java index c105d56c82..8e3a5d801f 100644 --- a/tika-app/src/test/java/org/apache/tika/cli/TikaCLITest.java +++ b/tika-app/src/test/java/org/apache/tika/cli/TikaCLITest.java @@ -36,10 +36,14 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.attribute.BasicFileAttributes; +import java.security.MessageDigest; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.HashSet; +import java.util.HexFormat; import java.util.List; +import java.util.Map; import java.util.Set; import com.fasterxml.jackson.databind.JsonNode; @@ -367,33 +371,34 @@ public class TikaCLITest { public void testRUnpack() throws Exception { //TODO -- rework this to use two separate emitters //one for bytes and one for json - // TODO: 00000001.bin extension may be wrong - see ~/Desktop/unpack-discussion/mime-todo.txt + // TODO: 1.bin extension may be wrong - see ~/Desktop/unpack-discussion/mime-todo.txt + // a single input file: the output dir is the package String[] expectedChildren = new String[]{ - "testPDFPackage.pdf.json", - //the first two test that the default single file config is working - "testPDFPackage.pdf/unpacked/00000001.bin", - "testPDFPackage.pdf/unpacked/00000002.jpg", - "testPDFPackage.pdf/unpacked/00000003.pdf", - "testPDFPackage.pdf/unpacked/00000004.pdf"}; - testRecursiveUnpack("testPDFPackage.pdf", expectedChildren, 2); + "datapackage.json", + "rmeta.json", + "unpacked/1.bin", + "unpacked/2.jpg", + "unpacked/3.pdf", + "unpacked/4.pdf"}; + testRecursiveUnpack("testPDFPackage.pdf", expectedChildren, 3); } @Test public void testPSTRUnpack() throws Exception { // TODO: The .bin extensions for embedded .msg files are wrong - they should be .msg // CONTENT_TYPE is not being set for embedded documents - see ~/Desktop/unpack-discussion/mime-todo.txt - String[] expectedChildren = new String[]{"testPST.pst.json", - "testPST.pst/unpacked/00000007.bin", - "testPST.pst/unpacked/00000001.bin", - "testPST.pst/unpacked/00000008.bin", - "testPST.pst/unpacked/00000004.bin", - "testPST.pst/unpacked/00000003.bin", - "testPST.pst/unpacked/00000002.bin", - "testPST.pst/unpacked/00000005.bin", - "testPST.pst/unpacked/00000009.docx", - "testPST.pst/unpacked/00000006.bin"}; - testRecursiveUnpack("testPST.pst", expectedChildren, 2); - try (Reader reader = Files.newBufferedReader(extractDir.resolve("testPST.pst.json"))) { + String[] expectedChildren = new String[]{"rmeta.json", + "unpacked/7.bin", + "unpacked/1.bin", + "unpacked/8.bin", + "unpacked/4.bin", + "unpacked/3.bin", + "unpacked/2.bin", + "unpacked/5.bin", + "unpacked/9.docx", + "unpacked/6.bin"}; + testRecursiveUnpack("testPST.pst", expectedChildren, 3); + try (Reader reader = Files.newBufferedReader(extractDir.resolve("rmeta.json"))) { List<Metadata> metadataList = JsonMetadataList.fromJson(reader); for (Metadata m : metadataList) { String content = m.get(TikaCoreProperties.TIKA_CONTENT); @@ -449,30 +454,30 @@ public class TikaCLITest { @Test public void testExtractSimple() throws Exception { - assertShallow("coffee.xls", 11, "coffee.xls.json", "coffee.xls/unpacked/00000001.emf", - "coffee.xls/unpacked/00000006.cdx", "coffee.xls/unpacked/00000005.png"); + assertShallow("coffee.xls", 10, "coffee.xls/rmeta.json", "coffee.xls/unpacked/1.emf", + "coffee.xls/unpacked/6.cdx", "coffee.xls/unpacked/5.png"); } @Test public void testExtractAbsolute() throws Exception { - assertShallow("testZip_absolutePath.zip", 5, "testZip_absolutePath.zip.json", - "testZip_absolutePath.zip/unpacked/00000001.bin"); + assertShallow("testZip_absolutePath.zip", 4, "testZip_absolutePath.zip/rmeta.json", + "testZip_absolutePath.zip/unpacked/1.bin"); } @Test public void testExtractRelative() throws Exception { - assertShallow("testZip_relative.zip", 4, "testZip_relative.zip.json"); + assertShallow("testZip_relative.zip", 3, "testZip_relative.zip/rmeta.json"); } @Test public void testExtractOverlapping() throws Exception { // overlapping names are handled by numbering - assertShallow("testZip_overlappingNames.zip", 5, "testZip_overlappingNames.zip.json"); + assertShallow("testZip_overlappingNames.zip", 4, "testZip_overlappingNames.zip/rmeta.json"); } @Test public void testExtract0x00() throws Exception { - assertShallow("testZip_zeroByte.zip", 4, "testZip_zeroByte.zip.json"); + assertShallow("testZip_zeroByte.zip", 3, "testZip_zeroByte.zip/rmeta.json"); } private void testRecursiveUnpack(String targetFile, String[] expectedChildrenFileNames) throws Exception { @@ -624,23 +629,84 @@ public class TikaCLITest { Set<String> fileNames = getFileNames(extractDir); // Should have extracted files in the specified directory, not current dir - assertTrue(fileNames.stream().anyMatch(f -> f.endsWith(".json")), - "Should have a .json metadata file in extractDir, got: " + fileNames); - assertTrue(fileNames.stream().anyMatch(f -> f.contains("/unpacked/")), + assertTrue(fileNames.contains("rmeta.json"), + "Should have rmeta.json in extractDir, got: " + fileNames); + assertTrue(fileNames.stream().anyMatch(f -> f.startsWith("unpacked/")), "Should have extracted embedded files in extractDir, got: " + fileNames); } - /** TIKA-4681: the default is a Frictionless package laid out as loose files. */ + /** + * TIKA-4681: the default is a Frictionless package laid out as loose files; for a single + * input file the output dir is the package (TIKA-4971). + */ @Test public void testDefaultIsALooseFrictionlessPackage() throws Exception { Set<String> fileNames = unpack("-Z"); - assertTrue(fileNames.stream().anyMatch(f -> f.endsWith("/datapackage.json")), - "package should be laid out as a directory, got: " + fileNames); - assertTrue(fileNames.stream().anyMatch(f -> f.contains("/unpacked/")), fileNames.toString()); - assertFalse(fileNames.stream().anyMatch(f -> f.endsWith(".zip") && !f.contains("/unpacked/")), + assertTrue(fileNames.contains("datapackage.json"), + "package should be laid out at the output dir's root, got: " + fileNames); + assertTrue(fileNames.contains("rmeta.json"), + "a Frictionless package carries rmeta.json by default: " + fileNames); + assertTrue(fileNames.stream().anyMatch(f -> f.matches("unpacked/1\\.[^/]+")), + "embedded files are unpadded and under unpacked/: " + fileNames); + assertFalse(fileNames.stream().anyMatch(f -> f.endsWith(".zip") && !f.startsWith("unpacked/")), "no zip when no mode was asked for: " + fileNames); - assertTrue(fileNames.stream().anyMatch(f -> f.endsWith("/metadata.json")), - "a Frictionless package carries metadata.json by default: " + fileNames); + assertFalse(fileNames.contains("test_recursive_embedded.docx.json"), + "rmeta.json already carries the metadata list: " + fileNames); + } + + /** TIKA-4971: rmeta.json is a resource of the package, with its size and hash. */ + @Test + public void testManifestListsRmetaJson() throws Exception { + unpack("-Z"); + byte[] metadataJson = Files.readAllBytes(extractDir.resolve("rmeta.json")); + JsonNode resources = new ObjectMapper().readTree( + extractDir.resolve("datapackage.json").toFile()).get("resources"); + JsonNode resource = null; + for (JsonNode r : resources) { + if ("rmeta.json".equals(r.get("path").asText())) { + resource = r; + } + } + assertNotNull(resource, "rmeta.json not in " + resources); + assertEquals("application/json", resource.get("mediatype").asText()); + assertEquals(metadataJson.length, resource.get("bytes").asLong()); + assertEquals("sha256:" + HexFormat.of().formatHex( + MessageDigest.getInstance("SHA-256").digest(metadataJson)), resource.get("hash").asText()); + } + + /** + * TIKA-4971: the rules frictionless validate enforces on resource names -- present, unique, + * lowercase [a-z0-9._-] -- and tk:embedded-id resolves to the file's row in rmeta.json. + */ + @Test + public void testManifestIsValidFrictionless() throws Exception { + unpack("-Z"); + ObjectMapper mapper = new ObjectMapper(); + JsonNode resources = mapper.readTree(extractDir.resolve("datapackage.json").toFile()) + .get("resources"); + Map<String, JsonNode> rowsById = new HashMap<>(); + for (JsonNode row : mapper.readTree(extractDir.resolve("rmeta.json").toFile())) { + if (row.has("tk:embedded-id")) { + rowsById.put(row.get("tk:embedded-id").asText(), row); + } + } + Set<String> names = new HashSet<>(); + int unpacked = 0; + for (JsonNode r : resources) { + String name = r.path("name").asText(null); + assertNotNull(name, "every resource needs a name: " + r); + assertTrue(name.matches("[a-z0-9._-]+"), "invalid Frictionless name: " + r); + assertTrue(names.add(name), "duplicate name: " + r); + if (!r.get("path").asText().startsWith("unpacked/")) { + continue; + } + unpacked++; + JsonNode row = rowsById.get(r.get("tk:embedded-id").asText()); + assertNotNull(row, "tk:embedded-id has no rmeta.json row: " + r); + assertEquals(row.path("tk:resource-name").asText(null), + r.path("tk:resource-name").asText(null), r.toString()); + } + assertTrue(unpacked > 0, resources.toString()); } @Test @@ -654,7 +720,7 @@ public class TikaCLITest { @Test public void testRegularFormatIsTheFlatLayout() throws Exception { Set<String> fileNames = unpack("-Z", "--unpack-format=REGULAR"); - assertTrue(fileNames.stream().anyMatch(f -> f.matches("test_recursive_embedded\\.docx-embed/0+1\\.[^/]+")), + assertTrue(fileNames.stream().anyMatch(f -> f.matches("test_recursive_embedded\\.docx-embed/1\\.[^/]+")), fileNames.toString()); assertFalse(fileNames.stream().anyMatch(f -> f.endsWith("datapackage.json") || f.contains("/unpacked/")), "REGULAR has no manifest and no unpacked/: " + fileNames); @@ -672,11 +738,10 @@ public class TikaCLITest { @Test public void testFrictionlessIncludeMetadata() throws Exception { Set<String> fileNames = unpack("-Z", "--unpack-include-metadata"); - String metadataJson = fileNames.stream().filter(f -> f.endsWith("/metadata.json")).findFirst() - .orElseThrow(() -> new AssertionError("no metadata.json in " + fileNames)); - String content = Files.readString(extractDir.resolve(metadataJson)); + assertTrue(fileNames.contains("rmeta.json"), "no rmeta.json in " + fileNames); + String content = Files.readString(extractDir.resolve("rmeta.json")); assertTrue(content.contains("\"tk:embedded-depth\" : \"0\""), - "metadata.json should carry the container row: " + content); + "rmeta.json should carry the container row: " + content); assertTrue(content.contains("\"tk:content\""), content); } @@ -684,9 +749,9 @@ public class TikaCLITest { @Test public void testConfigFileUnpackConfigIsHonoured() throws Exception { Set<String> fileNames = unpack("-Z", "--config=" + CONFIGS_DIR + "/tika-config-unpack-original.json"); - assertTrue(fileNames.stream().anyMatch(f -> f.matches(".*/unpacked/0+\\.[^/]+")), - "includeOriginal from -c should add the container as 00000000.<ext>: " + fileNames); - assertTrue(fileNames.stream().anyMatch(f -> f.matches(".*/unpacked/0+1\\.[^/]+")), + assertTrue(fileNames.stream().anyMatch(f -> f.matches("unpacked/0\\.[^/]+")), + "includeOriginal from -c should add the container as 0.<ext>: " + fileNames); + assertTrue(fileNames.stream().anyMatch(f -> f.matches("unpacked/1\\.[^/]+")), "-c must not change the CLI's file naming or packaging: " + fileNames); } diff --git a/tika-pipes/tika-async-cli/src/main/java/org/apache/tika/async/cli/TikaAsyncCLI.java b/tika-pipes/tika-async-cli/src/main/java/org/apache/tika/async/cli/TikaAsyncCLI.java index 9311b1abc6..96db5c1603 100644 --- a/tika-pipes/tika-async-cli/src/main/java/org/apache/tika/async/cli/TikaAsyncCLI.java +++ b/tika-pipes/tika-async-cli/src/main/java/org/apache/tika/async/cli/TikaAsyncCLI.java @@ -89,7 +89,7 @@ public class TikaAsyncCLI { options.addOption(null, "unpack-mode", true, "output mode for unpacking: DIRECTORY (default: loose files) or ZIPPED"); options.addOption(null, "unpack-include-metadata", false, - "metadata for every extracted file: a Frictionless package carries metadata.json " + "metadata for every extracted file: a Frictionless package carries rmeta.json " + "by default; this adds per-file sidecars to REGULAR zip output"); options.addOption(null, "on-exists", true, "behavior when an output file already exists: exception (default), replace or skip"); @@ -290,7 +290,13 @@ public class TikaAsyncCLI { inputDir = inString; // Only set default outputDir if not already specified via -o if (outputDir == null) { - if (Files.isRegularFile(inputPath)) { + if (Files.isRegularFile(inputPath) + && extractBytesMode != SimpleAsyncConfig.ExtractBytesMode.NONE + && isPackageDirectory(unpackFormat, unpackMode)) { + // the output dir is the package: keep it out of cwd + outputDir = Paths.get(inputPath.getFileName() + "-unpacked") + .toAbsolutePath().toString(); + } else if (Files.isRegularFile(inputPath)) { outputDir = Paths.get(".").toAbsolutePath().toString(); } else { outputDir = Paths.get("output").toAbsolutePath().toString(); @@ -311,6 +317,12 @@ public class TikaAsyncCLI { return config; } + /** Unpack output is a Frictionless package written as loose files. */ + private static boolean isPackageDirectory(String unpackFormat, String unpackMode) { + return (unpackFormat == null || UnpackConfig.OUTPUT_FORMAT.FRICTIONLESS.name().equals(unpackFormat)) + && (unpackMode == null || UnpackConfig.OUTPUT_MODE.DIRECTORY.name().equals(unpackMode)); + } + private static BasicContentHandlerFactory.HANDLER_TYPE getHandlerType(String t) throws TikaConfigException { return switch (t) { case "x" -> BasicContentHandlerFactory.HANDLER_TYPE.XML; @@ -402,7 +414,6 @@ public class TikaAsyncCLI { config.setEmitter(TikaConfigAsyncWriter.EMITTER_NAME); config.setSuffixStrategy(UnpackConfig.SUFFIX_STRATEGY.DETECTED); config.setEmbeddedIdPrefix("-"); - config.setZeroPadName(8); config.setKeyBaseStrategy(UnpackConfig.KEY_BASE_STRATEGY.DEFAULT); // Packaging is the CLI's too: loose files unless --unpack-mode asks for a zip @@ -419,6 +430,13 @@ public class TikaAsyncCLI { if (asyncConfig.isUnpackIncludeMetadata()) { config.setIncludeMetadata(true); } + // Single input file: the output dir is the package; one per input file otherwise + if (config.getOutputFormat() == UnpackConfig.OUTPUT_FORMAT.FRICTIONLESS + && config.getOutputMode() == UnpackConfig.OUTPUT_MODE.DIRECTORY + && asyncConfig.getInputDir() != null + && Files.isRegularFile(Paths.get(asyncConfig.getInputDir()))) { + config.setPackageRoot(""); + } parseContext.set(UnpackConfig.class, config); } diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/FrictionlessUnpackHandler.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/FrictionlessUnpackHandler.java index 0ceb8acc76..fd11bbf399 100644 --- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/FrictionlessUnpackHandler.java +++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/FrictionlessUnpackHandler.java @@ -160,7 +160,7 @@ public class FrictionlessUnpackHandler extends AbstractUnpackHandler implements DataPackage dataPackage = new DataPackage(containerName); // Add all embedded files with unpacked/ prefix - for (FrictionlessFileInfo fileInfo : embeddedFiles) { + for (FrictionlessFileInfo fileInfo : getEmbeddedFiles()) { String path = UNPACKED_DIR + "/" + fileInfo.fileName(); String originalName = fileInfo.metadata().get(TikaCoreProperties.RESOURCE_NAME_KEY); dataPackage.addResource(FrictionlessResource.create( @@ -168,6 +168,7 @@ public class FrictionlessUnpackHandler extends AbstractUnpackHandler implements fileInfo.mediatype(), fileInfo.bytes(), fileInfo.sha256Hash(), + fileInfo.id(), originalName )); } @@ -192,8 +193,11 @@ public class FrictionlessUnpackHandler extends AbstractUnpackHandler implements /** * Returns information about all embedded files. */ + /** In embedded-id order, not parse-completion order. */ public List<FrictionlessFileInfo> getEmbeddedFiles() { - return embeddedFiles; + return embeddedFiles.stream() + .sorted(java.util.Comparator.comparingInt(FrictionlessFileInfo::id)) + .toList(); } /** diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/UnpackConfig.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/UnpackConfig.java index bc02d5ce6a..5919f05fe0 100644 --- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/UnpackConfig.java +++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/UnpackConfig.java @@ -17,6 +17,7 @@ package org.apache.tika.pipes.core.extractor; import java.io.Serializable; +import java.util.Locale; import java.util.Objects; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -124,6 +125,13 @@ public class UnpackConfig implements Serializable { throw new IllegalArgumentException("can't parse OUTPUT_MODE: " + s); } } + /** FRICTIONLESS: a document with nothing to unpack still gets a package, or none. */ + public enum EMPTY_PACKAGE { + WRITE, + /** No package; tika-server's /unpack answers 204 */ + SKIP + } + private int zeroPadName = 0; private SUFFIX_STRATEGY suffixStrategy = SUFFIX_STRATEGY.NONE; @@ -151,6 +159,11 @@ public class UnpackConfig implements Serializable { // Frictionless Data Package options private OUTPUT_FORMAT outputFormat = OUTPUT_FORMAT.FRICTIONLESS; private OUTPUT_MODE outputMode = OUTPUT_MODE.ZIPPED; + // DIRECTORY-mode package root; null: the container's emit key + private String packageRoot = null; + // datapackage.json name; null: the fetch key's file name + private String packageName = null; + private EMPTY_PACKAGE emptyPackage = EMPTY_PACKAGE.WRITE; /** * Create an UnpackConfig with default settings. @@ -173,6 +186,9 @@ public class UnpackConfig implements Serializable { c.maxUnpackBytes = maxUnpackBytes; c.outputFormat = outputFormat; c.outputMode = outputMode; + c.packageRoot = packageRoot; + c.packageName = packageName; + c.emptyPackage = emptyPackage; return c; } @@ -264,7 +280,7 @@ public class UnpackConfig implements Serializable { } /** - * Metadata for every extracted file: {@code metadata.json} under FRICTIONLESS, a sidecar + * Metadata for every extracted file: {@code rmeta.json} under FRICTIONLESS, a sidecar * per file under REGULAR zip output, nothing under REGULAR loose output. Unset means on * for FRICTIONLESS, off for REGULAR. Not a bean getter: the class is introspected. */ @@ -368,6 +384,46 @@ public class UnpackConfig implements Serializable { setIncludeMetadata(includeFullMetadata); } + /** + * Emit key prefix of a FRICTIONLESS package in DIRECTORY mode. Unset: the container's + * emit key, one package per container. Empty: the emitter's root. + */ + public String getPackageRoot() { + return packageRoot; + } + + public void setPackageRoot(String packageRoot) { + this.packageRoot = packageRoot; + } + + /** + * {@code name} of a FRICTIONLESS package. Unset: the fetch key's file name. Blank: no + * {@code name} (tika-server sets the caller's filename, or blank when there is none). + */ + public String getPackageName() { + return packageName; + } + + public void setPackageName(String packageName) { + this.packageName = packageName; + } + + /** + * FRICTIONLESS: whether a document with nothing to unpack gets a package (its + * datapackage.json and rmeta.json) or none, which tika-server's /unpack answers with 204. + */ + public EMPTY_PACKAGE getEmptyPackage() { + return emptyPackage; + } + + public void setEmptyPackage(EMPTY_PACKAGE emptyPackage) { + this.emptyPackage = emptyPackage; + } + + public void setEmptyPackage(String emptyPackage) { + setEmptyPackage(EMPTY_PACKAGE.valueOf(emptyPackage.toUpperCase(Locale.ROOT))); + } + @Override public String toString() { return "UnpackConfig{" + "zeroPadName=" + zeroPadName + ", suffixStrategy=" + @@ -376,7 +432,8 @@ public class UnpackConfig implements Serializable { ", keyBaseStrategy=" + keyBaseStrategy + ", emitKeyBase='" + emitKeyBase + '\'' + ", zipEmbeddedFiles=" + zipEmbeddedFiles + ", includeMetadata=" + includeMetadata + ", maxUnpackBytes=" + maxUnpackBytes + ", outputFormat=" + outputFormat + - ", outputMode=" + outputMode + '}'; + ", outputMode=" + outputMode + ", packageRoot='" + packageRoot + '\'' + + ", packageName='" + packageName + '\'' + ", emptyPackage=" + emptyPackage + '}'; } @Override @@ -395,7 +452,10 @@ public class UnpackConfig implements Serializable { Objects.equals(includeMetadata, config.includeMetadata) && maxUnpackBytes == config.maxUnpackBytes && outputFormat == config.outputFormat && - outputMode == config.outputMode; + outputMode == config.outputMode && + Objects.equals(packageRoot, config.packageRoot) && + Objects.equals(packageName, config.packageName) && + emptyPackage == config.emptyPackage; } @Override @@ -412,6 +472,9 @@ public class UnpackConfig implements Serializable { result = 31 * result + Long.hashCode(maxUnpackBytes); result = 31 * result + Objects.hashCode(outputFormat); result = 31 * result + Objects.hashCode(outputMode); + result = 31 * result + Objects.hashCode(packageRoot); + result = 31 * result + Objects.hashCode(packageName); + result = 31 * result + Objects.hashCode(emptyPackage); return result; } } diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/frictionless/DataPackage.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/frictionless/DataPackage.java index 5078eaddf8..5c10280fb0 100644 --- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/frictionless/DataPackage.java +++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/frictionless/DataPackage.java @@ -31,6 +31,8 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; +import org.apache.tika.utils.StringUtils; + /** * Represents a Frictionless Data Package manifest (datapackage.json). * See: https://specs.frictionlessdata.io/data-package/ @@ -94,8 +96,9 @@ public class DataPackage { * @return sanitized name */ private static String sanitizeName(String name) { - if (name == null) { - return "unknown"; + // name is optional; an empty one is invalid + if (StringUtils.isBlank(name)) { + return null; } // Replace spaces, keep alphanumeric, dots, hyphens, underscores return name.toLowerCase(java.util.Locale.ROOT) @@ -152,19 +155,6 @@ public class DataPackage { this.resources.add(resource); } - /** - * Adds a resource to this data package with all parameters. - * - * @param path relative path within package - * @param mediatype MIME type - * @param bytes file size - * @param hash SHA256 hash with "sha256:" prefix - * @param name optional original filename - */ - public void addResource(String path, String mediatype, long bytes, String hash, String name) { - this.resources.add(FrictionlessResource.create(path, mediatype, bytes, hash, name)); - } - /** * Serializes this DataPackage to JSON. * diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/frictionless/FrictionlessResource.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/frictionless/FrictionlessResource.java index 600aacd8d5..39b37503da 100644 --- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/frictionless/FrictionlessResource.java +++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/frictionless/FrictionlessResource.java @@ -16,55 +16,54 @@ */ package org.apache.tika.pipes.core.extractor.frictionless; +import java.util.Locale; + import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; /** * Represents a resource entry in a Frictionless Data Package. * See: https://specs.frictionlessdata.io/data-resource/ * - * @param path The relative path to the file within the package (e.g., "unpacked/00000001.pdf") - * @param mediatype The MIME type of the file (e.g., "application/pdf") - * @param bytes The file size in bytes - * @param hash The SHA256 hash in format "sha256:hexstring" - * @param name Optional original filename if available + * @param name Frictionless resource name, derived from {@code path}: required, + * unique, lowercase {@code [a-z0-9._-]} + * @param path The relative path to the file within the package (e.g., "unpacked/1.pdf") + * @param mediatype The MIME type of the file (e.g., "application/pdf") + * @param bytes The file size in bytes + * @param hash The SHA256 hash in format "sha256:hexstring" + * @param embeddedId {@code tk:embedded-id} of the file's row in rmeta.json; 0 is the container + * @param resourceName {@code tk:resource-name}, the original filename, if there was one */ @JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({"name", "path", "mediatype", "bytes", "hash", "tk:embedded-id", "tk:resource-name"}) public record FrictionlessResource( + @JsonProperty("name") String name, @JsonProperty("path") String path, @JsonProperty("mediatype") String mediatype, @JsonProperty("bytes") long bytes, @JsonProperty("hash") String hash, - @JsonProperty("name") String name + @JsonProperty("tk:embedded-id") Integer embeddedId, + @JsonProperty("tk:resource-name") String resourceName ) { - /** - * Creates a FrictionlessResource with all fields. - * - * @param path relative path within package - * @param mediatype MIME type - * @param bytes file size - * @param hash SHA256 hash with "sha256:" prefix - * @param name optional original filename - * @return new FrictionlessResource - */ + /** A package file that is not an embedded document, e.g. rmeta.json. */ public static FrictionlessResource create(String path, String mediatype, long bytes, - String hash, String name) { - return new FrictionlessResource(path, mediatype, bytes, hash, name); + String hash) { + return new FrictionlessResource(nameFromPath(path), path, mediatype, bytes, hash, + null, null); } - /** - * Creates a FrictionlessResource without the optional name field. - * - * @param path relative path within package - * @param mediatype MIME type - * @param bytes file size - * @param hash SHA256 hash with "sha256:" prefix - * @return new FrictionlessResource - */ + /** An embedded document under unpacked/. */ public static FrictionlessResource create(String path, String mediatype, long bytes, - String hash) { - return new FrictionlessResource(path, mediatype, bytes, hash, null); + String hash, int embeddedId, String resourceName) { + return new FrictionlessResource(nameFromPath(path), path, mediatype, bytes, hash, + embeddedId, resourceName); + } + + /** Paths are unique within a package, so the names derived from them are too. */ + static String nameFromPath(String path) { + return path.toLowerCase(Locale.ROOT).replaceAll("[^a-z0-9._-]", "-"); } /** diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/EmitHandler.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/EmitHandler.java index 535820b223..feb57c402a 100644 --- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/EmitHandler.java +++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/EmitHandler.java @@ -73,8 +73,32 @@ class EmitHandler { this.directEmitThresholdBytes = directEmitThresholdBytes; } - public PipesResult emitParseData(FetchEmitTuple t, MetadataListAndEmbeddedBytes parseData, ParseContext parseContext) { - PipesResult result = emitParseDataInternal(t, parseData, parseContext); + /** Outcome of {@link #prepare}: the container stack trace and whether the tuple emits. */ + record Prepared(String stack, boolean emit) { + } + + /** + * Text inference, then the metadata filters, then user metadata, in place. Runs once per + * parse, before anything that writes the metadata list (the unpack package, the emit). + */ + Prepared prepare(FetchEmitTuple t, MetadataListAndEmbeddedBytes parseData, ParseContext parseContext) { + String stack = getContainerStacktrace(t, parseData.getMetadataList()); + boolean emit = StringUtils.isBlank(stack) || + t.getOnParseException() == FetchEmitTuple.ON_PARSE_EXCEPTION.EMIT; + if (emit) { + runTextInference(parseData, parseContext); + } + //we need to apply the metadata filter after we pull out the stacktrace + filterMetadata(parseData, parseContext); + if (emit) { + injectUserMetadata(t.getMetadata(), parseData.getMetadataList()); + } + return new Prepared(stack, emit); + } + + public PipesResult emitParseData(FetchEmitTuple t, MetadataListAndEmbeddedBytes parseData, + Prepared prepared, ParseContext parseContext) { + PipesResult result = emitParseDataInternal(t, parseData, prepared, parseContext); // Deadline exhaustion is a document-level fact, independent of which SUCCESS // variant (parsed-only, emitted, passed back...) the rest of this class chose -- // relabel any of them uniformly rather than special-casing every return site @@ -112,19 +136,10 @@ class EmitHandler { return parseRecord.isTaskDeadlineReached(); } - private PipesResult emitParseDataInternal(FetchEmitTuple t, MetadataListAndEmbeddedBytes parseData, ParseContext parseContext) { - long start = System.currentTimeMillis(); - String stack = getContainerStacktrace(t, parseData.getMetadataList()); - FetchEmitTuple.ON_PARSE_EXCEPTION onParseException = t.getOnParseException(); - boolean emit = StringUtils.isBlank(stack) || - onParseException == FetchEmitTuple.ON_PARSE_EXCEPTION.EMIT; - if (emit) { - runTextInference(parseData, parseContext); - } - //we need to apply the metadata filter after we pull out the stacktrace - filterMetadata(parseData, parseContext); - if (emit) { - injectUserMetadata(t.getMetadata(), parseData.getMetadataList()); + private PipesResult emitParseDataInternal(FetchEmitTuple t, MetadataListAndEmbeddedBytes parseData, + Prepared prepared, ParseContext parseContext) { + String stack = prepared.stack(); + if (prepared.emit()) { EmitKey emitKey = t.getEmitKey(); if (StringUtils.isBlank(emitKey.getEmitKey())) { emitKey = new EmitKey(emitKey.getEmitterId(), t.getFetchKey().getFetchKey()); @@ -210,6 +225,9 @@ class EmitHandler { ParseMode parseMode = parseContext.get(ParseMode.class); if (parseMode == ParseMode.CONTENT_ONLY && emitter instanceof StreamEmitter) { emitContentOnly((StreamEmitter) emitter, emitKey, parseData, parseContext); + } else if (parseData.isMetadataInPackage()) { + LOG.debug("metadata already emitted in the Frictionless package for {}", + emitKey.getEmitKey()); } else if (isExtractEmbeddedBytes && parseData.toBePackagedForStreamEmitter()) { emitContentsAndBytes(emitter, emitKey, parseData); diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/MetadataListAndEmbeddedBytes.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/MetadataListAndEmbeddedBytes.java index 5f371696fc..bb12b79a3b 100644 --- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/MetadataListAndEmbeddedBytes.java +++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/MetadataListAndEmbeddedBytes.java @@ -31,6 +31,7 @@ class MetadataListAndEmbeddedBytes { List<Metadata> metadataList; final UnpackHandler unpackHandler; + private boolean metadataInPackage = false; public MetadataListAndEmbeddedBytes(List<Metadata> metadataList, UnpackHandler unpackHandler) { @@ -60,6 +61,18 @@ class MetadataListAndEmbeddedBytes { return unpackHandler != null; } + /** + * True once a Frictionless package carrying this metadata list was emitted + * through the container's own emitter, so it must not be emitted again. + */ + public boolean isMetadataInPackage() { + return metadataInPackage; + } + + public void setMetadataInPackage(boolean metadataInPackage) { + this.metadataInPackage = metadataInPackage; + } + /** * If the intent is that the metadata and byte store be packaged in a zip * or similar and emitted via a single stream emitter. diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesWorker.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesWorker.java index da1f47600c..d5d9387112 100644 --- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesWorker.java +++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesWorker.java @@ -22,6 +22,8 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.List; import java.util.Set; import java.util.concurrent.Callable; @@ -60,6 +62,7 @@ import org.apache.tika.pipes.core.extractor.UnpackConfig; import org.apache.tika.pipes.core.extractor.UnpackExtractor; import org.apache.tika.pipes.core.extractor.UnpackedByteCount; import org.apache.tika.pipes.core.extractor.frictionless.DataPackage; +import org.apache.tika.pipes.core.extractor.frictionless.FrictionlessResource; import org.apache.tika.utils.ExceptionUtils; import org.apache.tika.utils.StringUtils; @@ -67,6 +70,8 @@ class PipesWorker implements Callable<PipesResult> { private static final Logger LOG = LoggerFactory.getLogger(PipesWorker.class); + static final String METADATA_FILE_NAME = "rmeta.json"; + private final FetchEmitTuple fetchEmitTuple; private final ParseContext parseContext; private final AutoDetectParser autoDetectParser; @@ -143,6 +148,9 @@ class PipesWorker implements Callable<PipesResult> { return PipesResults.EMPTY_OUTPUT; } + // before the unpack package: it carries the same metadata list as the emit + EmitHandler.Prepared prepared = emitHandler.prepare(fetchEmitTuple, parseData, parseContext); + // Check if we need to zip and emit embedded files UnpackHandler handler = parseContext.get(UnpackHandler.class); if (handler instanceof FrictionlessUnpackHandler) { @@ -163,7 +171,7 @@ class PipesWorker implements Callable<PipesResult> { long emitStart = System.nanoTime(); try { - return emitHandler.emitParseData(fetchEmitTuple, parseData, parseContext); + return emitHandler.emitParseData(fetchEmitTuple, parseData, prepared, parseContext); } finally { emitNanos = System.nanoTime() - emitStart; } @@ -265,11 +273,10 @@ class PipesWorker implements Callable<PipesResult> { */ private PipesResult emitFrictionlessOutput(FrictionlessUnpackHandler frictionlessHandler, MetadataListAndEmbeddedBytes parseData) { + // Written even with no embedded files, same shape on every surface, unless asked not to UnpackConfig unpackConfig = frictionlessHandler.getUnpackConfig(); - - // Skip if no embedded files (unless includeOriginal is set) - if (!frictionlessHandler.hasEmbeddedFiles() && !unpackConfig.isIncludeOriginal()) { - LOG.debug("No embedded files for Frictionless output"); + if (!frictionlessHandler.hasEmbeddedFiles() && + unpackConfig.getEmptyPackage() == UnpackConfig.EMPTY_PACKAGE.SKIP) { return null; } @@ -293,20 +300,62 @@ class PipesWorker implements Callable<PipesResult> { StreamEmitter streamEmitter = (StreamEmitter) emitter; EmitKey containerEmitKey = fetchEmitTuple.getEmitKey(); - // Get container name from fetch key - String fetchKey = fetchEmitTuple.getFetchKey().getFetchKey(); - String containerName = fetchKey; - int lastSlash = Math.max(fetchKey.lastIndexOf('/'), fetchKey.lastIndexOf('\\')); - if (lastSlash >= 0 && lastSlash < fetchKey.length() - 1) { - containerName = fetchKey.substring(lastSlash + 1); + // Package name: the host's, else the fetch key's file name + String containerName = unpackConfig.getPackageName(); + if (containerName == null) { + String fetchKey = fetchEmitTuple.getFetchKey().getFetchKey(); + containerName = fetchKey; + int lastSlash = Math.max(fetchKey.lastIndexOf('/'), fetchKey.lastIndexOf('\\')); + if (lastSlash >= 0 && lastSlash < fetchKey.length() - 1) { + containerName = fetchKey.substring(lastSlash + 1); + } } + DataPackage dataPackage = frictionlessHandler.buildDataPackage(containerName); + byte[] metadataJson = null; + if (unpackConfig.writesMetadata() && parseData != null && + parseData.getMetadataList() != null) { + try { + metadataJson = writeMetadataListAsJson(parseData.getMetadataList()); + } catch (IOException e) { + LOG.warn("Failed to serialize metadata for Frictionless output", e); + return new PipesResult(PipesResult.RESULT_STATUS.EMIT_EXCEPTION, + "Failed to serialize metadata for Frictionless output: " + + ExceptionUtils.format(e, parseContext)); + } + dataPackage.getResources().add(0, FrictionlessResource.create(METADATA_FILE_NAME, + "application/json", metadataJson.length, + FrictionlessResource.formatHash(sha256(metadataJson)))); + } + + PipesResult result; if (unpackConfig.getOutputMode() == UnpackConfig.OUTPUT_MODE.ZIPPED) { - return emitFrictionlessZipped(frictionlessHandler, streamEmitter, containerEmitKey, - containerName, parseData, unpackConfig); + result = emitFrictionlessZipped(frictionlessHandler, streamEmitter, containerEmitKey, + dataPackage, metadataJson); } else { - return emitFrictionlessDirectory(frictionlessHandler, streamEmitter, containerEmitKey, - containerName, parseData, unpackConfig); + result = emitFrictionlessDirectory(frictionlessHandler, streamEmitter, + packageRoot(containerEmitKey, unpackConfig), dataPackage, metadataJson); + } + // Same emitter: the package already carries the metadata list + if (result == null && metadataJson != null && + emitterName.equals(containerEmitKey.getEmitterId())) { + parseData.setMetadataInPackage(true); + } + return result; + } + + /** Emit key prefix of a directory-mode package; see {@link UnpackConfig#getPackageRoot()}. */ + private static String packageRoot(EmitKey containerEmitKey, UnpackConfig unpackConfig) { + String root = unpackConfig.getPackageRoot() != null ? + unpackConfig.getPackageRoot() : containerEmitKey.getEmitKey(); + return root.isEmpty() ? "" : root + "/"; + } + + private static byte[] sha256(byte[] bytes) { + try { + return MessageDigest.getInstance("SHA-256").digest(bytes); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 algorithm not available", e); } } @@ -316,16 +365,12 @@ class PipesWorker implements Callable<PipesResult> { private PipesResult emitFrictionlessZipped(FrictionlessUnpackHandler frictionlessHandler, StreamEmitter streamEmitter, EmitKey containerEmitKey, - String containerName, - MetadataListAndEmbeddedBytes parseData, - UnpackConfig unpackConfig) { - // Build the data package manifest - DataPackage dataPackage = frictionlessHandler.buildDataPackage(containerName); - + DataPackage dataPackage, + byte[] metadataJson) { // Create zip file in temp directory Path zipFile = frictionlessHandler.getTempDirectory().resolve("frictionless-package.zip"); try { - createFrictionlessZipFile(zipFile, frictionlessHandler, dataPackage, parseData, unpackConfig); + createFrictionlessZipFile(zipFile, frictionlessHandler, dataPackage, metadataJson); } catch (IOException e) { LOG.warn("Failed to create Frictionless zip file", e); return new PipesResult(PipesResult.RESULT_STATUS.EMIT_EXCEPTION, @@ -352,40 +397,31 @@ class PipesWorker implements Callable<PipesResult> { */ private PipesResult emitFrictionlessDirectory(FrictionlessUnpackHandler frictionlessHandler, StreamEmitter streamEmitter, - EmitKey containerEmitKey, - String containerName, - MetadataListAndEmbeddedBytes parseData, - UnpackConfig unpackConfig) { - String baseEmitKey = containerEmitKey.getEmitKey(); - - // Build the data package manifest - DataPackage dataPackage = frictionlessHandler.buildDataPackage(containerName); - + String packageRoot, + DataPackage dataPackage, + byte[] metadataJson) { try { // Emit each embedded file under unpacked/. // When includeOriginal=true the container itself is added as id 0 by // ParseHandler._preParse, so it appears here as one of the embedded entries. for (FrictionlessUnpackHandler.FrictionlessFileInfo fileInfo : frictionlessHandler.getEmbeddedFiles()) { - String fileEmitKey = baseEmitKey + "/unpacked/" + fileInfo.fileName(); + String fileEmitKey = packageRoot + "unpacked/" + fileInfo.fileName(); try (InputStream is = Files.newInputStream(fileInfo.filePath())) { streamEmitter.emit(fileEmitKey, is, fileInfo.metadata(), parseContext); } } // Emit datapackage.json - String dpEmitKey = baseEmitKey + "/datapackage.json"; + String dpEmitKey = packageRoot + "datapackage.json"; byte[] dpBytes = dataPackage.toJson().getBytes(java.nio.charset.StandardCharsets.UTF_8); try (InputStream dpStream = new java.io.ByteArrayInputStream(dpBytes)) { streamEmitter.emit(dpEmitKey, dpStream, new Metadata(), parseContext); } - // Emit metadata.json if requested - if (unpackConfig.writesMetadata() && parseData != null && - parseData.getMetadataList() != null) { - String metadataEmitKey = baseEmitKey + "/metadata.json"; - byte[] metadataBytes = writeMetadataListAsJson(parseData.getMetadataList()); - try (InputStream metadataStream = new java.io.ByteArrayInputStream(metadataBytes)) { - streamEmitter.emit(metadataEmitKey, metadataStream, new Metadata(), parseContext); + if (metadataJson != null) { + try (InputStream metadataStream = new java.io.ByteArrayInputStream(metadataJson)) { + streamEmitter.emit(packageRoot + METADATA_FILE_NAME, metadataStream, + new Metadata(), parseContext); } } } catch (IOException e) { @@ -395,7 +431,7 @@ class PipesWorker implements Callable<PipesResult> { } LOG.debug("Successfully emitted Frictionless package with {} resources (directory mode) to {}", - dataPackage.resourceCount(), baseEmitKey); + dataPackage.resourceCount(), packageRoot); return null; } @@ -403,8 +439,7 @@ class PipesWorker implements Callable<PipesResult> { * Creates a Frictionless Data Package zip file. */ private void createFrictionlessZipFile(Path zipFile, FrictionlessUnpackHandler frictionlessHandler, - DataPackage dataPackage, MetadataListAndEmbeddedBytes parseData, - UnpackConfig unpackConfig) throws IOException { + DataPackage dataPackage, byte[] metadataJson) throws IOException { try (ZipOutputStream zos = new ZipOutputStream(Files.newOutputStream(zipFile))) { // Add datapackage.json at root ZipEntry dpEntry = new ZipEntry("datapackage.json"); @@ -412,12 +447,9 @@ class PipesWorker implements Callable<PipesResult> { dataPackage.writeTo(zos); zos.closeEntry(); - // Add metadata.json if requested - if (unpackConfig.writesMetadata() && parseData != null && - parseData.getMetadataList() != null) { - ZipEntry metadataEntry = new ZipEntry("metadata.json"); - zos.putNextEntry(metadataEntry); - writeMetadataListAsJson(zos, parseData.getMetadataList()); + if (metadataJson != null) { + zos.putNextEntry(new ZipEntry(METADATA_FILE_NAME)); + zos.write(metadataJson); zos.closeEntry(); } diff --git a/tika-pipes/tika-pipes-core/src/main/resources/org/apache/tika/pipes/core/presets/thumbnail.json b/tika-pipes/tika-pipes-core/src/main/resources/org/apache/tika/pipes/core/presets/thumbnail.json index acb46b7333..e6e8c38d02 100644 --- a/tika-pipes/tika-pipes-core/src/main/resources/org/apache/tika/pipes/core/presets/thumbnail.json +++ b/tika-pipes/tika-pipes-core/src/main/resources/org/apache/tika/pipes/core/presets/thumbnail.json @@ -1,6 +1,8 @@ { // at most one raster thumbnail per parse, nothing else; the rule is in ThumbnailUnpackSelector "thumbnail-unpack-selector": {}, + // no thumbnail is no package: /unpack answers 204 + "unpack-config": {"emptyPackage": "SKIP"}, // every rendering parser: the top-level document's first page only, fitted into 256 x 256 in // colour, and text comes from the document itself, never OCR "pages": { diff --git a/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/extractor/frictionless/FrictionlessResourceTest.java b/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/extractor/frictionless/FrictionlessResourceTest.java new file mode 100644 index 0000000000..8e9c38d998 --- /dev/null +++ b/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/extractor/frictionless/FrictionlessResourceTest.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.pipes.core.extractor.frictionless; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +public class FrictionlessResourceTest { + + /** Frictionless resource names are lowercase [a-z0-9._-]; extensions in path are not. */ + @ParameterizedTest + @CsvSource(delimiter = '|', value = { + "rmeta.json|rmeta.json", + "unpacked/1.png|unpacked-1.png", + "unpacked/4.PDF|unpacked-4.pdf", + "unpacked/5.tar.gz|unpacked-5.tar.gz", + "unpacked/6.c++|unpacked-6.c--", + "unpacked/7.p df|unpacked-7.p-df", + "unpacked/8.\u00fcn|unpacked-8.-n" + }) + public void testNameFromPath(String path, String name) { + assertEquals(name, FrictionlessResource.nameFromPath(path)); + } +} diff --git a/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/core/FrictionlessUnpackTest.java b/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/core/FrictionlessUnpackTest.java index 19e6d7cc0a..33a2006ef6 100644 --- a/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/core/FrictionlessUnpackTest.java +++ b/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/core/FrictionlessUnpackTest.java @@ -63,7 +63,7 @@ import org.apache.tika.pipes.core.extractor.UnpackConfig; * <pre> * output/ * ├── datapackage.json # Frictionless manifest with file list, hashes, mimetypes - * ├── metadata.json # Full RMETA-style metadata (optional) + * ├── rmeta.json # Full RMETA-style metadata (optional) * └── unpacked/ * ├── 00000001.xml * ├── 00000002.xml @@ -149,7 +149,7 @@ public class FrictionlessUnpackTest { assertFalse(names.contains(TEST_DOC_WITH_EMBEDDED + "-embedded.zip"), names.toString()); Set<String> entries = zipEntries(onlyFrictionlessZip(outputDir)); assertTrue(entries.contains("datapackage.json"), entries.toString()); - assertTrue(entries.contains("metadata.json"), "a package carries metadata.json by default: " + entries); + assertTrue(entries.contains("rmeta.json"), "a package carries rmeta.json by default: " + entries); } @Test @@ -203,8 +203,8 @@ public class FrictionlessUnpackTest { "Each resource must have 'hash' (SHA256): " + resource); String path = resource.get("path").asText(); - assertTrue(path.startsWith("unpacked/"), - "Resource path should start with 'unpacked/': " + path); + assertTrue(path.startsWith("unpacked/") || path.equals("rmeta.json"), + "Resource path should be rmeta.json or under 'unpacked/': " + path); String hash = resource.get("hash").asText(); assertTrue(hash.startsWith("sha256:"), @@ -256,7 +256,7 @@ public class FrictionlessUnpackTest { @Test public void testIncludeMetadataJson() throws Exception { - // Test that includeFullMetadata=true creates metadata.json with RMETA-style output + // Test that includeFullMetadata=true creates rmeta.json with RMETA-style output ParseContext parseContext = frictionless(UnpackConfig.OUTPUT_MODE.ZIPPED, true); parseContext.get(UnpackConfig.class).setIncludeFullMetadata(true); PipesResult pipesResult = process("fullMetadata", TEST_DOC_WITH_EMBEDDED, parseContext); @@ -266,20 +266,20 @@ public class FrictionlessUnpackTest { Path zipFile = onlyFrictionlessZip(outputDir("fullMetadata")); Set<String> zipEntries = zipEntries(zipFile); - assertTrue(zipEntries.contains("metadata.json"), - "Zip should contain metadata.json when includeFullMetadata=true. Found: " + zipEntries); + assertTrue(zipEntries.contains("rmeta.json"), + "Zip should contain rmeta.json when includeFullMetadata=true. Found: " + zipEntries); JsonNode metadataJson; try (ZipFile zip = new ZipFile(zipFile.toFile()); - InputStream is = zip.getInputStream(zip.getEntry("metadata.json"))) { + InputStream is = zip.getInputStream(zip.getEntry("rmeta.json"))) { metadataJson = OBJECT_MAPPER.readTree(is); } - assertNotNull(metadataJson, "metadata.json should be parseable JSON"); + assertNotNull(metadataJson, "rmeta.json should be parseable JSON"); assertTrue(metadataJson.isArray(), - "metadata.json should be an array of metadata objects"); + "rmeta.json should be an array of metadata objects"); // container + 4 embedded assertTrue(metadataJson.size() >= 5, - "metadata.json should have at least 5 entries (container + 4 embedded). Found: " + + "rmeta.json should have at least 5 entries (container + 4 embedded). Found: " + metadataJson.size()); JsonNode containerMeta = metadataJson.get(0); @@ -299,7 +299,7 @@ public class FrictionlessUnpackTest { assertTrue(pipesResult.isSuccess(), "Processing with UnpackSelector should succeed"); - assertNothingPackaged(outputDir("selector"), TEST_DOC_WITH_EMBEDDED); + assertMetadataOnlyPackage(outputDir("selector"), TEST_DOC_WITH_EMBEDDED); } @Test @@ -334,7 +334,7 @@ public class FrictionlessUnpackTest { assertTrue(pipesResult.isSuccess(), "Frictionless should succeed with no embedded files"); - assertNothingPackaged(outputDir("noEmbedded"), SIMPLE_DOC); + assertMetadataOnlyPackage(outputDir("noEmbedded"), SIMPLE_DOC); } @Test @@ -368,8 +368,8 @@ public class FrictionlessUnpackTest { boolean manifestListsContainer = false; for (JsonNode resource : dataPackage.get("resources")) { String path = resource.get("path").asText(); - assertTrue(path.startsWith("unpacked/"), - "Manifest resources should only list unpacked/ paths; got " + path); + assertTrue(path.startsWith("unpacked/") || path.equals("rmeta.json"), + "Manifest resources should only list rmeta.json and unpacked/ paths; got " + path); if (path.equals("unpacked/0") || path.startsWith("unpacked/0.")) { manifestListsContainer = true; } @@ -379,6 +379,40 @@ public class FrictionlessUnpackTest { } } + /** + * TIKA-4971: rmeta.json gets what the emitted list gets -- the request's metadata filters, + * then user metadata, which the filters must not strip. + */ + @Test + public void testRmetaJsonIsFilteredThenGetsUserMetadata() throws Exception { + ParseContext parseContext = frictionless(UnpackConfig.OUTPUT_MODE.ZIPPED, false); + parseContext.setJsonConfig("metadata-filters", """ + [{"include-field-metadata-filter": {"include": ["tk:content", "Content-Type"]}}] + """); + Metadata userMetadata = new Metadata(); + userMetadata.set("my:key", "my-value"); + PipesResult pipesResult = pipesClient.process( + new FetchEmitTuple(TEST_DOC_WITH_EMBEDDED, + new FetchKey(FETCHER_NAME, TEST_DOC_WITH_EMBEDDED), + new EmitKey(EMITTER_NAME, "filtered/" + TEST_DOC_WITH_EMBEDDED), + userMetadata, parseContext, + FetchEmitTuple.ON_PARSE_EXCEPTION.EMIT)); + assertTrue(pipesResult.isSuccess(), "Status: " + pipesResult.status()); + + JsonNode rows; + try (ZipFile zip = new ZipFile(onlyFrictionlessZip(outputDir("filtered")).toFile()); + InputStream is = zip.getInputStream(zip.getEntry("rmeta.json"))) { + rows = OBJECT_MAPPER.readTree(is); + } + assertTrue(rows.size() > 1, rows.toString()); + Set<String> allowed = Set.of("tk:content", "Content-Type", "my:key"); + for (JsonNode row : rows) { + row.fieldNames().forEachRemaining(k -> + assertTrue(allowed.contains(k), "filter should have removed " + k + ": " + row)); + } + assertEquals("my-value", rows.get(0).get("my:key").asText(), rows.get(0).toString()); + } + private static Path onlyFrictionlessZip(Path outputDir) throws IOException { List<Path> zipFiles; try (Stream<Path> files = Files.list(outputDir)) { @@ -410,13 +444,18 @@ public class FrictionlessUnpackTest { return sb.toString(); } - /** Nothing to package writes no zip; the metadata JSON shows the parse still ran. */ - private static void assertNothingPackaged(Path outputDir, String emitKey) throws IOException { - try (Stream<Path> files = Files.list(outputDir)) { - List<String> zips = files.map(p -> p.getFileName().toString()) - .filter(n -> n.endsWith("-frictionless.zip")).toList(); - assertTrue(zips.isEmpty(), "unexpected zips: " + zips); + /** TIKA-4971: nothing to unpack is still a package: the manifest and rmeta.json, no sidecar. */ + private static void assertMetadataOnlyPackage(Path outputDir, String emitKey) throws IOException { + Path zipFile = outputDir.resolve(emitKey + "-frictionless.zip"); + assertEquals(Set.of("datapackage.json", "rmeta.json"), zipEntries(zipFile)); + JsonNode resources; + try (ZipFile zip = new ZipFile(zipFile.toFile()); + InputStream is = zip.getInputStream(zip.getEntry("datapackage.json"))) { + resources = OBJECT_MAPPER.readTree(is).get("resources"); } - assertTrue(Files.isRegularFile(outputDir.resolve(emitKey + ".json")), "no metadata JSON for " + emitKey); + assertEquals(1, resources.size(), resources.toString()); + assertEquals("rmeta.json", resources.get(0).get("path").asText()); + assertFalse(Files.exists(outputDir.resolve(emitKey + ".json")), + "rmeta.json already carries the metadata list"); } } diff --git a/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/core/UnpackModeTest.java b/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/core/UnpackModeTest.java index 755d10fe44..9c2d6d8d2e 100644 --- a/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/core/UnpackModeTest.java +++ b/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/core/UnpackModeTest.java @@ -17,6 +17,7 @@ package org.apache.tika.pipes.core; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -352,7 +353,7 @@ public class UnpackModeTest { Path outputDir = outputDir("zipNoEmbedded"); assertTrue(zips(outputDir).isEmpty(), "no embedded files, no zip: " + zips(outputDir)); - assertEquals(1, readMetadataList(outputDir, simpleDoc).size()); + assertEquals(1, readSidecar(outputDir, simpleDoc).size()); } @Test @@ -462,13 +463,25 @@ public class UnpackModeTest { return names; } - private static List<Metadata> readMetadataList(Path outputDir, String emitKey) throws IOException { + /** REGULAR: the metadata list is the {@code <emitKey>.json} beside the bytes. */ + private static List<Metadata> readSidecar(Path outputDir, String emitKey) throws IOException { try (Reader reader = Files.newBufferedReader(outputDir.resolve(emitKey + ".json"), StandardCharsets.UTF_8)) { return JsonMetadataList.fromJson(reader); } } + /** The package's rmeta.json: same emitter, so no separate {@code <emitKey>.json} (TIKA-4971). */ + private static List<Metadata> readMetadataList(Path outputDir, String emitKey) throws IOException { + assertFalse(Files.exists(outputDir.resolve(emitKey + ".json")), + "the package already carries the metadata list"); + try (ZipFile zip = new ZipFile(outputDir.resolve(emitKey + "-frictionless.zip").toFile()); + Reader reader = new java.io.InputStreamReader( + zip.getInputStream(zip.getEntry("rmeta.json")), StandardCharsets.UTF_8)) { + return JsonMetadataList.fromJson(reader); + } + } + /** The default package is a ZIPPED Frictionless one: entry name under unpacked/ -> size. */ private static Map<String, Long> embeddedFiles(Path outputDir, String emitKey) throws IOException { Path zipFile = outputDir.resolve(emitKey + "-frictionless.zip"); diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesParsingHelper.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesParsingHelper.java index be592413bf..e24c5ba4cd 100644 --- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesParsingHelper.java +++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesParsingHelper.java @@ -666,6 +666,9 @@ public class PipesParsingHelper { // Set emitter to our file-system emitter unpackConfig.setEmitter(UNPACK_EMITTER_ID); + // The fetch key may be the spool file; name the package for the caller's file + unpackConfig.setPackageName(callerSuppliedName == null ? "" : callerSuppliedName); + // /all = the original document plus metadata, in either output format. if (saveAll) { unpackConfig.setIncludeOriginal(true); diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java index 403f7977d1..0dbeb0bee8 100644 --- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java +++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java @@ -55,6 +55,7 @@ import org.apache.tika.config.JsonConfig; import org.apache.tika.config.OutputLimits; import org.apache.tika.config.loader.PresetRegistry; import org.apache.tika.config.loader.TikaLoader; +import org.apache.tika.config.loader.TikaObjectMapperFactory; import org.apache.tika.exception.TikaConfigException; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; @@ -215,6 +216,23 @@ public class TikaResource { return loadConfigDefaults().get(UnpackConfig.class); } + /** + * The config's unpack-config with the preset's unpack-config fields over it, field by + * field: the worker's preset merge replaces whole components, and the request's + * UnpackConfig (which the server always sends) would replace the preset's. Null when + * neither has one. + */ + public UnpackConfig newPresetUnpackConfig(String presetName) throws IOException { + UnpackConfig base = newConfigUnpackConfig(); + String presetJson = presetRegistry.parseContextJson(presetName); + ObjectMapper mapper = TikaObjectMapperFactory.getMapper(); + JsonNode presetUnpack = presetJson == null ? null : mapper.readTree(presetJson).get("unpack-config"); + if (presetUnpack == null) { + return base; + } + return mapper.readerForUpdating(base != null ? base : new UnpackConfig()).readValue(presetUnpack); + } + public TikaLoader getTikaLoader() { return tikaLoader; diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/UnpackerResource.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/UnpackerResource.java index cdf96de978..75af51a662 100644 --- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/UnpackerResource.java +++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/UnpackerResource.java @@ -18,6 +18,7 @@ package org.apache.tika.server.core.resource; import static org.apache.tika.server.core.resource.TikaResource.fillMetadata; +import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; @@ -96,7 +97,7 @@ import org.apache.tika.pipes.core.extractor.UnpackConfig; * <pre> * output.zip * ├── datapackage.json # Manifest with file list, SHA256 hashes, mimetypes - * ├── metadata.json # Full RMETA metadata (unless includeMetadata=false) + * ├── rmeta.json # Full RMETA metadata (unless includeMetadata=false) * └── unpacked/ * ├── 00000001.pdf * ├── 00000002.png @@ -291,6 +292,7 @@ public class UnpackerResource { @jakarta.ws.rs.PathParam("presetName") String presetName) throws Exception { ParseContext pc = tikaResource.createPresetContext(presetName); + seedPresetUnpackConfig(pc, presetName); rejectDirectoryMode(pc); Metadata metadata = tikaResource.newRequestMetadata(); try (TikaInputStream tis = TikaInputStream.get(is)) { @@ -310,6 +312,7 @@ public class UnpackerResource { @jakarta.ws.rs.PathParam("presetName") String presetName) throws Exception { ParseContext pc = tikaResource.createPresetContext(presetName); + seedPresetUnpackConfig(pc, presetName); rejectDirectoryMode(pc); Metadata metadata = tikaResource.newRequestMetadata(); try (TikaInputStream tis = TikaInputStream.get(is)) { @@ -319,6 +322,13 @@ public class UnpackerResource { } } + private void seedPresetUnpackConfig(ParseContext pc, String presetName) throws IOException { + UnpackConfig unpackConfig = tikaResource.newPresetUnpackConfig(presetName); + if (unpackConfig != null) { + pc.set(UnpackConfig.class, unpackConfig); + } + } + /** A request asking for DIRECTORY contradicts the one-zip transport: 400, not a silent pin. */ static void rejectDirectoryMode(ParseContext pc) { UnpackConfig requested = pc.get(UnpackConfig.class); diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/StackTraceTest.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/StackTraceTest.java index 80081c850b..06737f3adb 100644 --- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/StackTraceTest.java +++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/StackTraceTest.java @@ -183,13 +183,9 @@ public class StackTraceTest extends CXFTestBase { Response response = WebClient .create(endPoint + actualPath) .put(ClassLoader.getSystemResourceAsStream("test-documents/testDigilite.fdf")); - if (path.equals("/unpack")) { - //"NO CONTENT" - assertEquals(204, response.getStatus(), "bad type: " + actualPath); - } else { - assertEquals(200, response.getStatus(), "bad type: " + actualPath); - assertNotNull(response, "null response: " + actualPath); - } + // /unpack too: nothing to unpack is still a package (TIKA-4971) + assertEquals(200, response.getStatus(), "bad type: " + actualPath); + assertNotNull(response, "null response: " + actualPath); } } diff --git a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/ThumbnailPresetTest.java b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/ThumbnailPresetTest.java index 4ac707aa42..fdc72a5f3e 100644 --- a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/ThumbnailPresetTest.java +++ b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/ThumbnailPresetTest.java @@ -166,7 +166,7 @@ public class ThumbnailPresetTest extends CXFTestBase { /** * @return the embedded documents of the Frictionless package, without its - * datapackage.json and metadata.json + * datapackage.json and rmeta.json */ private Map<String, byte[]> unpack(String path, String file) throws IOException { Response response = put(path, fixture(file)); diff --git a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaPipesTest.java b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaPipesTest.java index f544e50c54..8a28798698 100644 --- a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaPipesTest.java +++ b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaPipesTest.java @@ -334,7 +334,7 @@ public class TikaPipesTest extends CXFTestBase { try (ZipFile zip = new ZipFile( outputBytesDir.resolve(TEST_RECURSIVE_DOC + "-frictionless.zip").toFile())) { assertNotNull(zip.getEntry("datapackage.json")); - assertNotNull(zip.getEntry("metadata.json")); + assertNotNull(zip.getEntry("rmeta.json")); for (Enumeration<? extends ZipEntry> e = zip.entries(); e.hasMoreElements(); ) { ZipEntry entry = e.nextElement(); if (entry.getName().startsWith("unpacked/")) { diff --git a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackFrictionlessTest.java b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackFrictionlessTest.java index 7c671b29e2..2d69ba48c3 100644 --- a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackFrictionlessTest.java +++ b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackFrictionlessTest.java @@ -116,15 +116,15 @@ public class UnpackFrictionlessTest extends CXFTestBase { assertTrue(entries.keySet().stream().anyMatch(k -> k.startsWith("unpacked/")), entries.keySet().toString()); assertTrue(entries.keySet().stream().allMatch(k -> k.startsWith("unpacked/") - || k.equals("datapackage.json") || k.equals("metadata.json")), + || k.equals("datapackage.json") || k.equals("rmeta.json")), "nothing but the package at the root: " + entries.keySet()); } - /** A Frictionless package carries metadata.json by default; /all adds only the original. */ + /** A Frictionless package carries rmeta.json by default; /all adds only the original. */ @Test public void testPlainUnpackHasMetadataJsonByDefault() throws Exception { Map<String, byte[]> entries = unpack("/unpack"); - assertTrue(entries.containsKey("metadata.json"), entries.keySet().toString()); + assertTrue(entries.containsKey("rmeta.json"), entries.keySet().toString()); assertFalse(entries.keySet().stream().anyMatch(k -> k.startsWith("unpacked/0.")), "plain /unpack must not pack the container's bytes: " + entries.keySet()); } @@ -147,7 +147,7 @@ public class UnpackFrictionlessTest extends CXFTestBase { @Test public void testUnpackAllCarriesMetadataAndTheOriginal() throws Exception { Map<String, byte[]> entries = unpack("/unpack/all"); - assertTrue(entries.containsKey("metadata.json"), entries.keySet().toString()); + assertTrue(entries.containsKey("rmeta.json"), entries.keySet().toString()); assertTrue(entries.keySet().stream().anyMatch(k -> k.startsWith("unpacked/0.")), "includeOriginal should add the container: " + entries.keySet()); } @@ -156,7 +156,7 @@ public class UnpackFrictionlessTest extends CXFTestBase { @Test public void testMetadataJsonCarriesTheContainerRow() throws Exception { Map<String, byte[]> entries = unpack("/unpack"); - JsonNode rows = MAPPER.readTree(entries.get("metadata.json")); + JsonNode rows = MAPPER.readTree(entries.get("rmeta.json")); assertTrue(rows.isArray() && rows.size() > 1, "expected an RMETA list, got: " + rows); JsonNode container = rows.get(0); @@ -172,7 +172,7 @@ public class UnpackFrictionlessTest extends CXFTestBase { @Test public void testRowsCarryEmbeddedPathData() throws Exception { Map<String, byte[]> entries = unpack("/unpack/all"); - JsonNode rows = MAPPER.readTree(entries.get("metadata.json")); + JsonNode rows = MAPPER.readTree(entries.get("rmeta.json")); JsonNode embedded = rows.get(1); for (String key : new String[]{"tk:embedded-id", "tk:embedded-id-path", "tk:embedded-resource-path", "tk:resource-name"}) { diff --git a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackRegularConfigTest.java b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackRegularConfigTest.java index 07bf63c0aa..290182a8f5 100644 --- a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackRegularConfigTest.java +++ b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackRegularConfigTest.java @@ -113,7 +113,7 @@ public class UnpackRegularConfigTest extends CXFTestBase { public void testServerConfigRestoresTheFlatLayout() throws Exception { Map<String, byte[]> entries = unpack("/unpack"); assertFalse(entries.containsKey("datapackage.json"), entries.keySet().toString()); - assertFalse(entries.containsKey("metadata.json"), entries.keySet().toString()); + assertFalse(entries.containsKey("rmeta.json"), entries.keySet().toString()); assertTrue(entries.keySet().stream().noneMatch(k -> k.contains("/")), "REGULAR puts every file at the zip root: " + entries.keySet()); assertFalse(entries.isEmpty()); diff --git a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackerResourceHandlerTest.java b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackerResourceHandlerTest.java index c1169f14ee..34be6f5275 100644 --- a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackerResourceHandlerTest.java +++ b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackerResourceHandlerTest.java @@ -53,9 +53,9 @@ import org.apache.tika.server.core.writer.ZipWriter; /** * /unpack has no format segment, so the config part is the only place a handler is named -- - * and it reaches the RMETA list written to metadata.json. + * and it reaches the RMETA list written to rmeta.json. * Per-file sidecars exist only in the REGULAR layout, so the server config pins it; the - * metadata.json probes ask for FRICTIONLESS explicitly. + * rmeta.json probes ask for FRICTIONLESS explicitly. */ public class UnpackerResourceHandlerTest extends CXFTestBase { @@ -131,7 +131,7 @@ public class UnpackerResourceHandlerTest extends CXFTestBase { String json = metadataJson("{" + FRICTIONLESS + ", \"basic-content-handler-factory\": {\"type\": \"XML\"}}"); assertTrue(json.contains("<html xmlns="), - "config part should select the handler for metadata.json"); + "config part should select the handler for rmeta.json"); } /** /unpack/all/<handler> used to 200 and silently ignore the segment. */ @@ -155,11 +155,11 @@ public class UnpackerResourceHandlerTest extends CXFTestBase { public void testAllConfigSpellingIsNotAHandlerNamedConfig() throws Exception { Response response = post("/unpack/all/config", "{" + FRICTIONLESS + "}"); assertEquals(200, response.getStatus()); - assertNotNull(readZipArchiveBytes((InputStream) response.getEntity()).get("metadata.json")); + assertNotNull(readZipArchiveBytes((InputStream) response.getEntity()).get("rmeta.json")); response = post("/unpack/all/config/ignore", "{" + FRICTIONLESS + "}"); assertEquals(200, response.getStatus()); String json = new String(readZipArchiveBytes((InputStream) response.getEntity()) - .get("metadata.json"), StandardCharsets.UTF_8); + .get("rmeta.json"), StandardCharsets.UTF_8); assertFalse(json.contains("\"tk:content\""), "ignore after /config must suppress tk:content"); } @@ -256,8 +256,8 @@ public class UnpackerResourceHandlerTest extends CXFTestBase { Response response = post("/unpack", configJson); assertEquals(200, response.getStatus()); Map<String, byte[]> entries = readZipArchiveBytes((InputStream) response.getEntity()); - byte[] md = entries.get("metadata.json"); - assertNotNull(md, "includeFullMetadata should write metadata.json"); + byte[] md = entries.get("rmeta.json"); + assertNotNull(md, "includeFullMetadata should write rmeta.json"); return new String(md, StandardCharsets.UTF_8); } diff --git a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackerResourceTest.java b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackerResourceTest.java index 26ffa7680f..0c5e37a3ec 100644 --- a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackerResourceTest.java +++ b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackerResourceTest.java @@ -172,8 +172,8 @@ public class UnpackerResourceTest extends CXFTestBase { .toList(); assertEquals(2, wavFiles.size(), "Should have 2 WAV files"); - assertTrue(data.containsKey("metadata.json"), - "/all carries the parse as metadata.json. Entries: " + data.keySet()); + assertTrue(data.containsKey("rmeta.json"), + "/all carries the parse as rmeta.json. Entries: " + data.keySet()); } @Test @@ -221,10 +221,41 @@ public class UnpackerResourceTest extends CXFTestBase { assertTrue(data.containsValue(DOCX_IMAGE2_MD5), "Should contain second DOCX image"); } + /** TIKA-4971: a request can ask for no package when there is nothing to unpack -- a 204. */ @Test - @org.junit.jupiter.api.Disabled("Behavior changed in Tika 4.0 pipes-based implementation") - public void test204() throws Exception { - // This test verified 204 behavior which changed in 4.0 + public void testEmptyPackageSkipIs204() throws Exception { + String configJson = """ + {"parse-context": {"unpack-config": {"emptyPackage": "SKIP"}}} + """; + ContentDisposition fileCd = new ContentDisposition("form-data; name=\"file\"; filename=\"foo.csv\""); + Attachment fileAtt = new Attachment("file", + ClassLoader.getSystemResourceAsStream("test-documents/foo.csv"), fileCd); + Attachment configAtt = new Attachment("config", "application/json", + new ByteArrayInputStream(configJson.getBytes(StandardCharsets.UTF_8))); + Response response = WebClient + .create(endPoint + UNPACKER_PATH) + .type("multipart/form-data") + .accept("application/zip") + .post(new MultipartBody(Arrays.asList(fileAtt, configAtt))); + assertEquals(204, response.getStatus()); + } + + /** TIKA-4971: no attachments is still a package -- the manifest and rmeta.json, not a 204. */ + @Test + public void testNoAttachmentsIsAMetadataOnlyPackage() throws Exception { + Response response = WebClient + .create(endPoint + UNPACKER_PATH) + .header("Content-Disposition", "attachment; filename=\"foo.csv\"") + .accept("application/zip") + .put(ClassLoader.getSystemResourceAsStream("test-documents/foo.csv")); + + assertEquals(200, response.getStatus()); + Map<String, byte[]> data = readZipArchiveBytes((InputStream) response.getEntity()); + assertEquals(Set.of("datapackage.json", "rmeta.json"), data.keySet()); + JsonNode dataPackage = MAPPER.readTree(data.get("datapackage.json")); + assertEquals("foo.csv", dataPackage.get("name").asText()); + assertEquals(1, dataPackage.get("resources").size(), dataPackage.toString()); + assertEquals("rmeta.json", dataPackage.get("resources").get(0).get("path").asText()); } @Test @@ -276,8 +307,8 @@ public class UnpackerResourceTest extends CXFTestBase { Map<String, byte[]> data = readZipArchiveBytes((InputStream) response.getEntity()); - byte[] metadata = data.get("metadata.json"); - assertNotNull(metadata, "Should have metadata.json. Entries: " + data.keySet()); + byte[] metadata = data.get("rmeta.json"); + assertNotNull(metadata, "Should have rmeta.json. Entries: " + data.keySet()); String metadataJson = new String(metadata, StandardCharsets.UTF_8); assertTrue(metadataJson.contains("Content-Type"), "Metadata JSON should contain Content-Type"); } @@ -458,9 +489,8 @@ public class UnpackerResourceTest extends CXFTestBase { /** * The datapackage.json "resources" array is the manifest for the Frictionless * package. Verifies it lists exactly the data files present in the zip, with - * no missing or extraneous entries. The package envelope (datapackage.json - * and the optional metadata.json) is not itself a "resource" and is excluded - * from the comparison. + * no missing or extraneous entries. Only datapackage.json, the manifest itself, + * is not a resource; rmeta.json is (TIKA-4971). */ @Test public void testFrictionlessDataPackageMatchesArchiveContents() throws Exception { @@ -507,7 +537,6 @@ public class UnpackerResourceTest extends CXFTestBase { Set<String> archiveDataFiles = new HashSet<>(data.keySet()); archiveDataFiles.remove("datapackage.json"); - archiveDataFiles.remove("metadata.json"); assertEquals(archiveDataFiles, manifestPaths, "datapackage.json 'resources' must list exactly the data files in the zip. " + @@ -516,9 +545,9 @@ public class UnpackerResourceTest extends CXFTestBase { } /** - * The Frictionless manifest's "name" field is supposed to carry the - * original filename of each resource. For the container (unpacked/0.<ext>), - * that name should be the filename the user supplied on the multipart + * The manifest's "tk:resource-name" carries the original filename of each resource + * ("name" is the Frictionless identifier, derived from the path). For the container + * (unpacked/0.<ext>), it should be the filename the user supplied on the multipart * upload -- not the server's internal spool filename. */ @Test @@ -566,9 +595,10 @@ public class UnpackerResourceTest extends CXFTestBase { "Manifest should list the container at unpacked/0. Resources: " + dataPackage.get("resources")); - JsonNode nameNode = containerResource.get("name"); + assertEquals(0, containerResource.get("tk:embedded-id").asInt(), containerResource.toString()); + JsonNode nameNode = containerResource.get("tk:resource-name"); assertNotNull(nameNode, - "Container resource should carry a 'name' field. Resource: " + containerResource); + "Container resource should carry 'tk:resource-name'. Resource: " + containerResource); assertEquals(uploadFilename, nameNode.asText(), "Container's manifest name should be the user-supplied upload filename, " + "not the server's internal spool filename. Resource: " + containerResource); @@ -616,7 +646,7 @@ public class UnpackerResourceTest extends CXFTestBase { /** * Documents the shape of /unpack output in FRICTIONLESS mode (no /all, - * default config): a datapackage.json manifest, the metadata.json envelope + * default config): a datapackage.json manifest, the rmeta.json envelope * a Frictionless package carries by default, and the unpacked/<id> children. */ @Test @@ -648,8 +678,8 @@ public class UnpackerResourceTest extends CXFTestBase { assertTrue(data.containsKey("datapackage.json"), "Should contain datapackage.json manifest. Entries: " + data.keySet()); - assertTrue(data.containsKey("metadata.json"), - "A Frictionless package carries metadata.json by default. Entries: " + data.keySet()); + assertTrue(data.containsKey("rmeta.json"), + "A Frictionless package carries rmeta.json by default. Entries: " + data.keySet()); boolean hasUnpacked = data.keySet().stream().anyMatch(k -> k.startsWith("unpacked/")); assertTrue(hasUnpacked, "Should contain unpacked/ entries. Entries: " + data.keySet()); } @@ -716,7 +746,7 @@ public class UnpackerResourceTest extends CXFTestBase { /** * Documents that includeFullMetadata=true in FRICTIONLESS adds a - * metadata.json envelope whose entries carry tk:content (the extracted + * rmeta.json envelope whose entries carry tk:content (the extracted * text) and Content-Type alongside the per-document metadata. */ @Test @@ -746,13 +776,13 @@ public class UnpackerResourceTest extends CXFTestBase { assertEquals(200, response.getStatus()); Map<String, byte[]> data = readZipArchiveBytes((InputStream) response.getEntity()); - byte[] metadataBytes = data.get("metadata.json"); - assertNotNull(metadataBytes, "metadata.json should be present when includeFullMetadata=true. " + + byte[] metadataBytes = data.get("rmeta.json"); + assertNotNull(metadataBytes, "rmeta.json should be present when includeFullMetadata=true. " + "Entries: " + data.keySet()); JsonNode metadata = MAPPER.readTree(metadataBytes); assertTrue(metadata.isArray() && metadata.size() > 0, - "metadata.json should be a non-empty array"); + "rmeta.json should be a non-empty array"); JsonNode container = metadata.get(0); assertTrue(container.has("Content-Type"),
