This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new c42b10873c add shim so that tika-eval works for 4.0.0 vs 4.0.0-beta-1
(#2975)
c42b10873c is described below
commit c42b10873ce7cfede66a5315dc2ca2baba1a39c4
Author: Tim Allison <[email protected]>
AuthorDate: Wed Jul 29 21:38:48 2026 -0400
add shim so that tika-eval works for 4.0.0 vs 4.0.0-beta-1 (#2975)
---
.../org/apache/tika/eval/app/io/ExtractReader.java | 76 ++++++++++++++++++++++
.../apache/tika/eval/app/io/ExtractReaderTest.java | 34 ++++++++++
.../test-dirs/legacy/beta1-style.doc.json | 17 +++++
.../resources/test-dirs/legacy/collision.doc.json | 7 ++
4 files changed, 134 insertions(+)
diff --git
a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/io/ExtractReader.java
b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/io/ExtractReader.java
index 26ff7f22cf..bac92b48e4 100644
---
a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/io/ExtractReader.java
+++
b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/io/ExtractReader.java
@@ -25,8 +25,10 @@ import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import java.util.Locale;
+import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -159,6 +161,9 @@ public class ExtractReader {
try {
if (fileSuffixes.format == FileSuffixes.FORMAT.JSON) {
metadataList = JsonMetadataList.fromJson(reader);
+ for (Metadata m : metadataList) {
+ normalizeLegacyKeys(m);
+ }
if (alterMetadataList.equals(ALTER_METADATA_LIST.FIRST_ONLY)
&& metadataList.size() > 1) {
while (metadataList.size() > 1) {
metadataList.remove(metadataList.size() - 1);
@@ -214,6 +219,77 @@ public class ExtractReader {
}
+ // Pre-4.0 extract key -> 4.0 key, for the Tika-native fields tika-eval
reads. Digest keys are
+ // handled by the prefix rule in normalizeLegacyKeys;
Content-Type/Content-Length are standard
+ // names (unchanged) so they are not listed. New-side keys come from the
live constants so this
+ // can't drift from the 4.0 declarations.
+ private static final Map<String, String> LEGACY_KEY_MAP = Map.ofEntries(
+ Map.entry("X-TIKA:content",
TikaCoreProperties.TIKA_CONTENT.getName()),
+ Map.entry("X-TIKA:content_handler",
TikaCoreProperties.TIKA_CONTENT_HANDLER.getName()),
+ Map.entry("X-TIKA:embedded_depth",
TikaCoreProperties.EMBEDDED_DEPTH.getName()),
+ Map.entry("X-TIKA:embedded_resource_path",
TikaCoreProperties.EMBEDDED_RESOURCE_PATH.getName()),
+ Map.entry("X-TIKA:final_embedded_resource_path",
TikaCoreProperties.FINAL_EMBEDDED_RESOURCE_PATH.getName()),
+ Map.entry("X-TIKA:parse_time_millis",
TikaCoreProperties.PARSE_TIME_MILLIS.getName()),
+ Map.entry("X-TIKA:resourceName",
TikaCoreProperties.RESOURCE_NAME_KEY.getName()),
+ Map.entry("X-TIKA:detectedEncoding",
TikaCoreProperties.DETECTED_ENCODING.getName()),
+ Map.entry("X-TIKA:encodingDetector",
TikaCoreProperties.ENCODING_DETECTOR.getName()),
+ Map.entry("Content-Type-Hint",
TikaCoreProperties.CONTENT_TYPE_HINT.getName()),
+ Map.entry("embeddedResourceType",
TikaCoreProperties.EMBEDDED_RESOURCE_TYPE.getName()),
+ Map.entry("X-TIKA:EXCEPTION:container_exception",
TikaCoreProperties.CONTAINER_EXCEPTION.getName()),
+ Map.entry("X-TIKA:EXCEPTION:embedded_exception",
TikaCoreProperties.EMBEDDED_EXCEPTION.getName()));
+
+ private static final String LEGACY_DIGEST_PREFIX =
TikaCoreProperties.LEGACY_TIKA_META_PREFIX
+ + "digest" + TikaCoreProperties.NAMESPACE_PREFIX_DELIMITER;
+
+ /**
+ * Pre-4.0 extracts (e.g. 4.0.0-beta-1) key Tika-native fields under
X-TIKA:/camelCase names.
+ * Normalize the fields tika-eval reads to their 4.0 tk: keys so a
cross-version compare reflects
+ * real diffs, not the rename. Harmless on 4.0 extracts: the legacy keys
are simply absent.
+ */
+ private static void normalizeLegacyKeys(Metadata m) {
+ m.setTrusted(true); // sanctioned trusted transformation: may write
reserved tk: keys
+ try {
+ for (Map.Entry<String, String> e : LEGACY_KEY_MAP.entrySet()) {
+ remapLegacyKey(m, e.getKey(), e.getValue());
+ }
+ // digest keys: X-TIKA:digest:<alg> -> tk:digest:<alg> (algorithm
unchanged; MD5 drives
+ // embedded-doc matching). names() is a snapshot, so remapping
while iterating is safe.
+ for (String name : m.names()) {
+ if (name.startsWith(LEGACY_DIGEST_PREFIX)) {
+ remapLegacyKey(m, name, TikaCoreProperties.TIKA_META_PREFIX
+ +
name.substring(TikaCoreProperties.LEGACY_TIKA_META_PREFIX.length()));
+ }
+ }
+ } finally {
+ m.setTrusted(false);
+ }
+ }
+
+ private static void remapLegacyKey(Metadata m, String legacyKey, String
modernKey) {
+ String[] legacyVals = m.getValues(legacyKey);
+ if (legacyVals.length == 0) {
+ return;
+ }
+ String[] modernVals = m.getValues(modernKey);
+ if (modernVals.length > 0) {
+ // Both present: safe only if identical. Fail loud rather than
silently clobber a value.
+ if (!Arrays.equals(legacyVals, modernVals)) {
+ throw new IllegalStateException("Extract has both legacy key
'" + legacyKey
+ + "' and modern key '" + modernKey + "' with different
values; legacy-key "
+ + "normalization would clobber. Extract is
inconsistent.");
+ }
+ } else {
+ for (int i = 0; i < legacyVals.length; i++) {
+ if (i == 0) {
+ m.set(modernKey, legacyVals[i]);
+ } else {
+ m.add(modernKey, legacyVals[i]);
+ }
+ }
+ }
+ m.remove(legacyKey);
+ }
+
public enum ALTER_METADATA_LIST {
AS_IS, //leave the metadata list as is
FIRST_ONLY, //take only the metadata list for the "container" document
diff --git
a/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/io/ExtractReaderTest.java
b/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/io/ExtractReaderTest.java
index 8c33d42ffc..e164bfe4fa 100644
---
a/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/io/ExtractReaderTest.java
+++
b/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/io/ExtractReaderTest.java
@@ -18,6 +18,8 @@ package org.apache.tika.eval.app.io;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import java.io.IOException;
import java.nio.file.Path;
@@ -88,6 +90,38 @@ public class ExtractReaderTest extends TikaTest {
.get(TikaCoreProperties.TIKA_CONTENT));
}
+ @Test
+ public void testLegacyKeyNormalization() throws Exception {
+ // a pre-4.0 (4.0.0-beta-1 style) extract: X-TIKA:/camelCase
Tika-native keys
+ Path f =
getResourceAsFile("/test-dirs/legacy/beta1-style.doc.json").toPath();
+ List<Metadata> list = new ExtractReader().loadExtract(f);
+ assertEquals(2, list.size());
+
+ Metadata container = list.get(0);
+ // readable via the 4.0 Property constants that tika-eval uses
+ assertEquals("the quick brown fox",
container.get(TikaCoreProperties.TIKA_CONTENT));
+ assertEquals("12",
container.get(TikaCoreProperties.PARSE_TIME_MILLIS));
+ assertEquals("boom",
container.get(TikaCoreProperties.CONTAINER_EXCEPTION));
+ assertEquals("abc123", container.get("tk:digest:MD5"));
+ // the legacy keys are gone
+ assertNull(container.get("X-TIKA:content"));
+ assertNull(container.get("X-TIKA:digest:MD5"));
+
+ Metadata embedded = list.get(1);
+ assertEquals("inner.txt",
embedded.get(TikaCoreProperties.EMBEDDED_RESOURCE_PATH));
+ assertEquals("1", embedded.get(TikaCoreProperties.EMBEDDED_DEPTH));
+ assertEquals("inner.txt",
embedded.get(TikaCoreProperties.RESOURCE_NAME_KEY));
+ assertEquals("ATTACHMENT",
embedded.get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE));
+ assertNull(embedded.get("X-TIKA:embedded_resource_path"));
+ }
+
+ @Test
+ public void testLegacyNormalizationCollisionFailsLoud() throws Exception {
+ // both the legacy and modern content key present with different
values -> must not clobber
+ Path f =
getResourceAsFile("/test-dirs/legacy/collision.doc.json").toPath();
+ assertThrows(IllegalStateException.class, () -> new
ExtractReader().loadExtract(f));
+ }
+
@Test
public void testTextBasic() throws IOException {
ExtractReader extractReader = new ExtractReader();
diff --git
a/tika-eval/tika-eval-app/src/test/resources/test-dirs/legacy/beta1-style.doc.json
b/tika-eval/tika-eval-app/src/test/resources/test-dirs/legacy/beta1-style.doc.json
new file mode 100644
index 0000000000..d8938f4a06
--- /dev/null
+++
b/tika-eval/tika-eval-app/src/test/resources/test-dirs/legacy/beta1-style.doc.json
@@ -0,0 +1,17 @@
+[
+ {
+ "Content-Type" : "text/plain",
+ "X-TIKA:content" : "the quick brown fox",
+ "X-TIKA:parse_time_millis" : "12",
+ "X-TIKA:digest:MD5" : "abc123",
+ "X-TIKA:EXCEPTION:container_exception" : "boom"
+ },
+ {
+ "Content-Type" : "text/plain",
+ "X-TIKA:embedded_resource_path" : "inner.txt",
+ "X-TIKA:content" : "attachment contents",
+ "X-TIKA:embedded_depth" : "1",
+ "X-TIKA:resourceName" : "inner.txt",
+ "embeddedResourceType" : "ATTACHMENT"
+ }
+]
diff --git
a/tika-eval/tika-eval-app/src/test/resources/test-dirs/legacy/collision.doc.json
b/tika-eval/tika-eval-app/src/test/resources/test-dirs/legacy/collision.doc.json
new file mode 100644
index 0000000000..47c2b342e1
--- /dev/null
+++
b/tika-eval/tika-eval-app/src/test/resources/test-dirs/legacy/collision.doc.json
@@ -0,0 +1,7 @@
+[
+ {
+ "Content-Type" : "text/plain",
+ "X-TIKA:content" : "legacy value",
+ "tk:content" : "modern value"
+ }
+]
