This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 62fe6fcf74 TIKA-4969: opt-in typed metadata values in JSON output
(#3341)
62fe6fcf74 is described below
commit 62fe6fcf74321c55e9fed1853d99c090be5eb72f
Author: Tim Allison <[email protected]>
AuthorDate: Mon Oct 19 14:29:31 2026 +0100
TIKA-4969: opt-in typed metadata values in JSON output (#3341)
---
CHANGES.txt | 6 +
docs/modules/ROOT/pages/configuration/index.adoc | 39 ++-
.../pages/migration-to-4x/chunk-strategies.adoc | 4 +-
.../java/org/apache/tika/metadata/Property.java | 18 +-
.../apache/tika/metadata/TikaCoreProperties.java | 2 +-
.../org/apache/tika/metadata/metadata-keys.json | 2 +-
.../core/serialization/EmitDataDeserializer.java | 18 ++
.../core/serialization/EmitDataSerializer.java | 17 ++
.../core/serialization/EmitDataValueTypesTest.java | 85 +++++++
.../org/apache/tika/pipes/emitter/es/ESClient.java | 15 +-
.../pipes/emitter/opensearch/OpenSearchClient.java | 16 +-
.../org/apache/tika/config/loader/TikaLoader.java | 25 +-
.../apache/tika/serialization/JsonMetadata.java | 38 ++-
.../tika/serialization/JsonMetadataList.java | 38 ++-
.../tika/serialization/MetadataValueTypes.java | 94 +++++++
.../serialization/serdes/MetadataDeserializer.java | 64 ++++-
.../serialization/serdes/MetadataSerializer.java | 102 +++++++-
.../tika/serialization/MetadataValueTypesTest.java | 97 ++++++++
.../serialization/TypedMetadataValuesTest.java | 272 +++++++++++++++++++++
19 files changed, 906 insertions(+), 46 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 16556bfc19..1c02d2fd92 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,11 @@
Release 4.2.0 - unreleased
+ * Opt-in "metadata-list": {"typedValues": true} writes INTEGER, REAL and
+ BOOLEAN metadata as JSON numbers and booleans, and tk:chunks as nested
+ JSON rather than an escaped string, in -J, /rmeta and the JSON emitters.
+ New Property.ValueType.JSON; tk:chunks is now JSON-typed. Readers accept
+ both forms (TIKA-4969).
+
* The docx parser wraps OMML equations (m:oMath) in <span class="math">;
their
text was previously emitted as unmarked body text (TIKA-4968).
diff --git a/docs/modules/ROOT/pages/configuration/index.adoc
b/docs/modules/ROOT/pages/configuration/index.adoc
index 0421974bc5..77a0c40ea4 100644
--- a/docs/modules/ROOT/pages/configuration/index.adoc
+++ b/docs/modules/ROOT/pages/configuration/index.adoc
@@ -57,7 +57,7 @@ optional; anything you omit uses its defaults.
"pipes-iterator": { /* iterator (one per pipeline) */ },
"pipes-reporters": { /* per-document status reporters */ },
"plugin-roots": "/path/to/plugins",
- "metadata-list": { /* Jackson read limits for metadata JSON */ },
+ "metadata-list": { /* metadata JSON: Jackson read limits, typedValues */ },
"service-loader": { /* SPI load-failure policy */ },
"xml-reader-utils": { /* XML parser pool size */ }
}
@@ -80,8 +80,8 @@ Unknown keys are rejected almost everywhere: at the top
level, inside a componen
and inside a `FetchEmitTuple`. Four places swallow them silently instead, so a
misspelling there
produces no error and no effect:
-* the `metadata-list` body — only `maxStringLength`, `maxNestingDepth` and
`maxNumberLength` are
- read;
+* the `metadata-list` body — only `maxStringLength`, `maxNestingDepth`,
`maxNumberLength` and
+ `typedValues` are read;
* the body of a self-configuring component under `parse-context` — it is
handed to the component
untouched;
* `_mime-include` / `_mime-exclude` when the value is neither a JSON array nor
a string — a bare
@@ -94,6 +94,39 @@ produces no error and no effect:
If a setting appears to do nothing, check its spelling against one of those
four first.
+[#typed-values]
+=== Typed values in metadata JSON (`typedValues`)
+
+By default every metadata value in JSON output (`tika-app -J`, tika-server
`/rmeta`, and the
+file-system, S3, GCS and Azure emitters) is a string, so `tk:chunks` arrives
as an escaped
+string that consumers must parse a second time. Opt in to native types:
+
+[source,json]
+----
+{ "metadata-list": { "typedValues": true } }
+----
+
+Values of keys registered as `INTEGER` or `REAL` are then written as JSON
numbers, `BOOLEAN`
+as `true`/`false`, and `JSON` keys such as `tk:chunks` as nested JSON:
+
+[source,json]
+----
+{ "audio:channels": 2, "audio:has-drm": false, "tk:chunks": [ { "text": "...",
... } ] }
+----
+
+The key's type comes from its `Property` declaration; `tika-metadata-schema`'s
+`metadata-keys.json` lists it as `valueType`. A value that does not fit its
type (`"abc"` on an
+`INTEGER` key, a JSON value with more than one value on the key) stays a
string, as does a JSON
+value nested deeper than `maxNestingDepth` allows, so the output always reads
back. Only a
+single-valued object, or an array holding an object or array, is written as
JSON: a JSON scalar
+or an array of scalars would be indistinguishable from an ordinary value or
values. Tika's own
+readers accept either form; it is off by default because the change in types
breaks consumers
+that expect strings, and an index that maps a key from its first value (as
Elasticsearch does)
+rejects a later document whose value for that key is malformed and stays a
string.
+
+The Elasticsearch and OpenSearch emitters always write `JSON` keys as JSON,
whatever this
+setting.
+
[#component-names]
== Component names
diff --git a/docs/modules/ROOT/pages/migration-to-4x/chunk-strategies.adoc
b/docs/modules/ROOT/pages/migration-to-4x/chunk-strategies.adoc
index 84888fa275..7b2aa0a414 100644
--- a/docs/modules/ROOT/pages/migration-to-4x/chunk-strategies.adoc
+++ b/docs/modules/ROOT/pages/migration-to-4x/chunk-strategies.adoc
@@ -28,7 +28,9 @@ are kept for earlier shapes.
== The `tk:chunks` Field
-All chunk data lands in one metadata field, `tk:chunks`, as a JSON array.
There is no
+All chunk data lands in one metadata field, `tk:chunks`, as a JSON array (in
JSON output, an
+escaped string unless
+xref:configuration/index.adoc#typed-values[`metadata-list.typedValues`] is
on). There is no
separate field for image embeddings versus text embeddings — a chunk is a
chunk, identified
by which locators it carries:
diff --git a/tika-core/src/main/java/org/apache/tika/metadata/Property.java
b/tika-core/src/main/java/org/apache/tika/metadata/Property.java
index 10d3394ad4..bbb414a3c8 100644
--- a/tika-core/src/main/java/org/apache/tika/metadata/Property.java
+++ b/tika-core/src/main/java/org/apache/tika/metadata/Property.java
@@ -255,6 +255,12 @@ public final class Property implements
Comparable<Property> {
return new Property(name, true, ValueType.RATIONAL);
}
+ /** A single value holding one JSON document; see {@link ValueType#JSON}.
*/
+ public static Property internalJson(String name) {
+ requireNotReserved(name);
+ return new Property(name, true, ValueType.JSON);
+ }
+
public static Property internalOpenChoice(String name, String... choices) {
requireNotReserved(name);
return new Property(name, true, ValueType.OPEN_CHOICE, choices);
@@ -377,6 +383,11 @@ public final class Property implements
Comparable<Property> {
return new Property(name, true, ValueType.RATIONAL);
}
+ static Property reservedInternalJson(String name) {
+ requireReserved(name);
+ return new Property(name, true, ValueType.JSON);
+ }
+
static Property reservedInternalText(String name) {
requireReserved(name);
return new Property(name, true, ValueType.TEXT);
@@ -563,7 +574,12 @@ public final class Property implements
Comparable<Property> {
public enum ValueType {
BOOLEAN, OPEN_CHOICE, CLOSED_CHOICE, DATE, INTEGER,
- RATIONAL, REAL, TEXT, URI, PROPERTY
+ RATIONAL, REAL, TEXT, URI, PROPERTY,
+ /**
+ * A serialized JSON document. Not an XMP type: Tika's own, for
structured values
+ * such as chunks. Stored as a String like any other value.
+ */
+ JSON
}
}
diff --git
a/tika-core/src/main/java/org/apache/tika/metadata/TikaCoreProperties.java
b/tika-core/src/main/java/org/apache/tika/metadata/TikaCoreProperties.java
index d1587b1873..797be9fa10 100644
--- a/tika-core/src/main/java/org/apache/tika/metadata/TikaCoreProperties.java
+++ b/tika-core/src/main/java/org/apache/tika/metadata/TikaCoreProperties.java
@@ -151,7 +151,7 @@ public interface TikaCoreProperties {
* Used by inference parsers and metadata filters to attach chunked
representations
* of document content for downstream indexing and semantic search.
*/
- Property TIKA_CHUNKS = Property.reservedInternalText("tk:chunks");
+ Property TIKA_CHUNKS = Property.reservedInternalJson("tk:chunks");
/**
* The inference inputs a parser released for this document (TEXT, PAGES,
...), written
diff --git
a/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-keys.json
b/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-keys.json
index 3f57a2f533..cac32492ea 100644
---
a/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-keys.json
+++
b/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-keys.json
@@ -553,7 +553,7 @@
{"key":"tiff:Software","namespace":"tiff","valueType":"TEXT","cardinality":"SIMPLE","module":"tika-core"},
{"key":"tiff:XResolution","namespace":"tiff","valueType":"RATIONAL","cardinality":"SIMPLE","module":"tika-core"},
{"key":"tiff:YResolution","namespace":"tiff","valueType":"RATIONAL","cardinality":"SIMPLE","module":"tika-core"},
-
{"key":"tk:chunks","namespace":"tk","valueType":"TEXT","cardinality":"SIMPLE","module":"tika-core"},
+
{"key":"tk:chunks","namespace":"tk","valueType":"JSON","cardinality":"SIMPLE","module":"tika-core"},
{"key":"tk:content","namespace":"tk","valueType":"TEXT","cardinality":"SIMPLE","module":"tika-core"},
{"key":"tk:content-handler-type","namespace":"tk","valueType":"TEXT","cardinality":"SIMPLE","module":"tika-core"},
{"key":"tk:content-type-hint","namespace":"tk","valueType":"TEXT","cardinality":"SIMPLE","module":"tika-core"},
diff --git
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/EmitDataDeserializer.java
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/EmitDataDeserializer.java
index 1f63fa9325..741149720c 100644
---
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/EmitDataDeserializer.java
+++
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/EmitDataDeserializer.java
@@ -22,7 +22,9 @@ import static
org.apache.tika.pipes.core.serialization.EmitDataSerializer.METADA
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.databind.DeserializationContext;
@@ -32,6 +34,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.pipes.core.emitter.EmitDataImpl;
+import org.apache.tika.serialization.MetadataValueTypes;
import org.apache.tika.utils.StringUtils;
public class EmitDataDeserializer extends JsonDeserializer<EmitDataImpl> {
@@ -42,6 +45,7 @@ public class EmitDataDeserializer extends
JsonDeserializer<EmitDataImpl> {
ObjectMapper mapper = (ObjectMapper) jsonParser.getCodec();
String emitKey = readString(EMIT_KEY, root, null, true);
+ learnValueTypes(root);
List<Metadata> metadataList = readMetadataList(root, mapper);
String containerStackTrace = readString(CONTAINER_STACK_TRACE, root,
StringUtils.EMPTY, false);
@@ -54,6 +58,20 @@ public class EmitDataDeserializer extends
JsonDeserializer<EmitDataImpl> {
return emitData;
}
+ private static void learnValueTypes(JsonNode root) {
+ JsonNode types = root.get(EmitDataSerializer.VALUE_TYPES);
+ if (types == null || !types.isObject()) {
+ return;
+ }
+ Iterator<Map.Entry<String, JsonNode>> fields = types.fields();
+ while (fields.hasNext()) {
+ Map.Entry<String, JsonNode> e = fields.next();
+ if (e.getValue().isTextual()) {
+ MetadataValueTypes.learn(e.getKey(), e.getValue().textValue());
+ }
+ }
+ }
+
private static List<Metadata> readMetadataList(JsonNode root, ObjectMapper
mapper) throws IOException {
JsonNode metadataListNode = root.get(METADATA_LIST);
if (metadataListNode == null || !metadataListNode.isArray()) {
diff --git
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/EmitDataSerializer.java
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/EmitDataSerializer.java
index 5f7b87b397..502cddd820 100644
---
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/EmitDataSerializer.java
+++
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/EmitDataSerializer.java
@@ -17,12 +17,16 @@
package org.apache.tika.pipes.core.serialization;
import java.io.IOException;
+import java.util.Map;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.JsonSerializer;
import com.fasterxml.jackson.databind.SerializerProvider;
+import org.apache.tika.metadata.Property;
import org.apache.tika.pipes.api.emitter.EmitData;
+import org.apache.tika.serialization.JsonMetadataList;
+import org.apache.tika.serialization.MetadataValueTypes;
import org.apache.tika.utils.StringUtils;
public class EmitDataSerializer extends JsonSerializer<EmitData> {
@@ -31,12 +35,25 @@ public class EmitDataSerializer extends
JsonSerializer<EmitData> {
public static final String METADATA_LIST = "metadataList";
public static final String CONTAINER_STACK_TRACE = "containerStackTrace";
public static final String CONTENT_BYTES = "contentBytes";
+ public static final String VALUE_TYPES = "valueTypes";
@Override
public void serialize(EmitData emitData, JsonGenerator jsonGenerator,
SerializerProvider serializerProvider) throws IOException {
jsonGenerator.writeStartObject();
jsonGenerator.writeStringField(EMIT_KEY, emitData.getEmitKey());
jsonGenerator.writeObjectField(METADATA_LIST,
emitData.getMetadataList());
+ // the parent may never load the classes that declare these keys
+ if (JsonMetadataList.isTypedValues() && emitData.getMetadataList() !=
null) {
+ Map<String, Property.ValueType> types =
+ MetadataValueTypes.describe(emitData.getMetadataList());
+ if (!types.isEmpty()) {
+ jsonGenerator.writeObjectFieldStart(VALUE_TYPES);
+ for (Map.Entry<String, Property.ValueType> e :
types.entrySet()) {
+ jsonGenerator.writeStringField(e.getKey(),
e.getValue().name());
+ }
+ jsonGenerator.writeEndObject();
+ }
+ }
if (!StringUtils.isBlank(emitData.getContainerStackTrace())) {
jsonGenerator.writeStringField(CONTAINER_STACK_TRACE,
emitData.getContainerStackTrace());
}
diff --git
a/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/serialization/EmitDataValueTypesTest.java
b/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/serialization/EmitDataValueTypesTest.java
new file mode 100644
index 0000000000..057eef8fa6
--- /dev/null
+++
b/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/serialization/EmitDataValueTypesTest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.core.serialization;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+import java.io.StringWriter;
+import java.util.List;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+
+import org.apache.tika.metadata.Audio;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.pipes.core.emitter.EmitDataImpl;
+import org.apache.tika.serialization.JsonMetadataList;
+
+/**
+ * A pipes parent serializes metadata whose keys are declared by classes only
the fork loaded,
+ * so the fork sends their types along.
+ */
+public class EmitDataValueTypesTest {
+
+ private static final ObjectMapper SMILE = new ObjectMapper(new
SmileFactory());
+
+ @AfterEach
+ public void reset() {
+ JsonMetadataList.setTypedValues(false);
+ }
+
+ @Test
+ public void testForkReportsTypedKeys() throws Exception {
+ JsonMetadataList.setTypedValues(true);
+ JsonNode types =
SMILE.readTree(JsonPipesIpc.toBytes(emitData())).get("valueTypes");
+ assertEquals("{\"audio:channels\":\"INTEGER\"}", types.toString());
+ }
+
+ @Test
+ public void testNoTypesWhenUntyped() throws Exception {
+
assertNull(SMILE.readTree(JsonPipesIpc.toBytes(emitData())).get("valueTypes"));
+ }
+
+ @Test
+ public void testParentWritesReportedTypes() throws Exception {
+ // a key this JVM does not declare, as a plugin-declared key is in
tika-server
+ String key = "test-emit-value-types:count";
+ ObjectNode wire = SMILE.createObjectNode();
+ wire.put("emitKey", "key");
+ wire.putArray("metadataList").addObject().put(key, "7");
+ wire.putObject("valueTypes").put(key, "INTEGER");
+ EmitDataImpl back =
JsonPipesIpc.fromBytes(SMILE.writeValueAsBytes(wire), EmitDataImpl.class);
+
+ JsonMetadataList.setTypedValues(true);
+ StringWriter writer = new StringWriter();
+ JsonMetadataList.toJson(back.getMetadataList(), writer);
+ assertEquals(7, new
ObjectMapper().readTree(writer.toString()).get(0).get(key).intValue());
+ }
+
+ private static EmitDataImpl emitData() {
+ Metadata m = new Metadata();
+ m.set(Audio.CHANNELS, "2");
+ m.set(TikaCoreProperties.TITLE, "t");
+ return new EmitDataImpl("key", List.of(m));
+ }
+}
diff --git
a/tika-pipes/tika-pipes-plugins/tika-pipes-es/src/main/java/org/apache/tika/pipes/emitter/es/ESClient.java
b/tika-pipes/tika-pipes-plugins/tika-pipes-es/src/main/java/org/apache/tika/pipes/emitter/es/ESClient.java
index 4652fa7a7e..3c32f6c413 100644
---
a/tika-pipes/tika-pipes-plugins/tika-pipes-es/src/main/java/org/apache/tika/pipes/emitter/es/ESClient.java
+++
b/tika-pipes/tika-pipes-plugins/tika-pipes-es/src/main/java/org/apache/tika/pipes/emitter/es/ESClient.java
@@ -29,7 +29,6 @@ import java.util.Base64;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.UUID;
import com.fasterxml.jackson.core.JsonFactory;
@@ -50,7 +49,9 @@ import org.slf4j.LoggerFactory;
import org.apache.tika.client.TikaClientException;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.Property;
import org.apache.tika.pipes.api.emitter.EmitData;
+import org.apache.tika.serialization.MetadataValueTypes;
import org.apache.tika.utils.StringUtils;
/**
@@ -486,19 +487,19 @@ public class ESClient {
}
/**
- * Metadata fields whose values are serialized JSON from the
- * tika-inference pipeline. These must be written as raw JSON
- * (arrays/objects) rather than escaped strings so that
- * ES can index vectors, locators, etc. natively.
+ * Keys declared {@link Property.ValueType#JSON}, e.g. {@code tk:chunks}:
written as JSON
+ * rather than escaped strings so that ES can index vectors, locators,
etc. natively.
*/
- static final Set<String> INFERENCE_JSON_FIELDS = Set.of("tk:chunks");
+ static boolean isJsonField(String name) {
+ return MetadataValueTypes.get(name) == Property.ValueType.JSON;
+ }
private static void writeMetadata(Metadata metadata, JsonGenerator
jsonGenerator)
throws IOException {
for (String n : metadata.names()) {
String[] vals = metadata.getValues(n);
if (vals.length == 1) {
- if (INFERENCE_JSON_FIELDS.contains(n) && isValidJson(vals[0]))
{
+ if (isJsonField(n) && isValidJson(vals[0])) {
jsonGenerator.writeFieldName(n);
jsonGenerator.writeRawValue(vals[0]);
} else {
diff --git
a/tika-pipes/tika-pipes-plugins/tika-pipes-opensearch/src/main/java/org/apache/tika/pipes/emitter/opensearch/OpenSearchClient.java
b/tika-pipes/tika-pipes-plugins/tika-pipes-opensearch/src/main/java/org/apache/tika/pipes/emitter/opensearch/OpenSearchClient.java
index 66c7609d97..18e1705517 100644
---
a/tika-pipes/tika-pipes-plugins/tika-pipes-opensearch/src/main/java/org/apache/tika/pipes/emitter/opensearch/OpenSearchClient.java
+++
b/tika-pipes/tika-pipes-plugins/tika-pipes-opensearch/src/main/java/org/apache/tika/pipes/emitter/opensearch/OpenSearchClient.java
@@ -29,7 +29,6 @@ import java.util.Base64;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.UUID;
import com.fasterxml.jackson.core.JsonFactory;
@@ -49,7 +48,9 @@ import org.slf4j.LoggerFactory;
import org.apache.tika.client.TikaClientException;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.Property;
import org.apache.tika.pipes.api.emitter.EmitData;
+import org.apache.tika.serialization.MetadataValueTypes;
import org.apache.tika.utils.StringUtils;
public class OpenSearchClient {
@@ -468,13 +469,12 @@ public class OpenSearchClient {
}
/**
- * Metadata fields whose values are serialized JSON from the
- * tika-inference pipeline. These must be written as raw JSON
- * (arrays/objects) rather than escaped strings so that
- * OpenSearch can index vectors, locators, etc. natively.
+ * Keys declared {@link Property.ValueType#JSON}, e.g. {@code tk:chunks}:
written as JSON
+ * rather than escaped strings so that OpenSearch can index vectors,
locators, etc. natively.
*/
- static final Set<String> INFERENCE_JSON_FIELDS = Set.of(
- "tk:chunks");
+ static boolean isJsonField(String name) {
+ return MetadataValueTypes.get(name) == Property.ValueType.JSON;
+ }
private static void writeMetadata(Metadata metadata, JsonGenerator
jsonGenerator) throws IOException {
//writes the metadata without the start { or the end }
@@ -482,7 +482,7 @@ public class OpenSearchClient {
for (String n : metadata.names()) {
String[] vals = metadata.getValues(n);
if (vals.length == 1) {
- if (INFERENCE_JSON_FIELDS.contains(n)
+ if (isJsonField(n)
&& isValidJson(vals[0])) {
jsonGenerator.writeFieldName(n);
jsonGenerator.writeRawValue(vals[0]);
diff --git
a/tika-serialization/src/main/java/org/apache/tika/config/loader/TikaLoader.java
b/tika-serialization/src/main/java/org/apache/tika/config/loader/TikaLoader.java
index ef9b422016..faa05736ed 100644
---
a/tika-serialization/src/main/java/org/apache/tika/config/loader/TikaLoader.java
+++
b/tika-serialization/src/main/java/org/apache/tika/config/loader/TikaLoader.java
@@ -581,7 +581,7 @@ public class TikaLoader {
*
* <p>Settings include:
* <ul>
- * <li>metadata-list - Jackson StreamReadConstraints for
JsonMetadata/JsonMetadataList serialization</li>
+ * <li>metadata-list - Jackson StreamReadConstraints and typedValues for
JsonMetadata/JsonMetadataList serialization</li>
* <li>service-loader - Service loader configuration</li>
* <li>xml-reader-utils - XML parser security settings</li>
* </ul>
@@ -636,22 +636,39 @@ public class TikaLoader {
* Loads the metadata-list configuration section and applies it to
* JsonMetadata and JsonMetadataList serializers.
* <p>
- * Configuration uses Jackson's StreamReadConstraints property names:
+ * Read limits use Jackson's StreamReadConstraints property names; {@code
typedValues}
+ * opts in to native numbers, booleans and JSON values (see
+ * {@link JsonMetadataList#setTypedValues(boolean)}):
* <pre>
* {
* "metadata-list": {
* "maxStringLength": 20000000,
* "maxNestingDepth": 10,
- * "maxNumberLength": 500
+ * "maxNumberLength": 500,
+ * "typedValues": true
* }
* }
* </pre>
*/
- private void loadMetadataListConfig() {
+ private void loadMetadataListConfig() throws TikaConfigException {
JsonNode metadataListNode = config.getRootNode().get("metadata-list");
if (metadataListNode == null) {
return;
}
+ if (metadataListNode.has("typedValues")) {
+ JsonNode typedNode = metadataListNode.get("typedValues");
+ if (!typedNode.isBoolean()) {
+ throw new TikaConfigException(
+ "metadata-list.typedValues must be true or false, not
" + typedNode);
+ }
+ boolean typed = typedNode.booleanValue();
+ JsonMetadata.setTypedValues(typed);
+ JsonMetadataList.setTypedValues(typed);
+ }
+ if (!metadataListNode.has("maxStringLength") &&
!metadataListNode.has("maxNestingDepth")
+ && !metadataListNode.has("maxNumberLength")) {
+ return;
+ }
StreamReadConstraints.Builder builder =
StreamReadConstraints.builder();
diff --git
a/tika-serialization/src/main/java/org/apache/tika/serialization/JsonMetadata.java
b/tika-serialization/src/main/java/org/apache/tika/serialization/JsonMetadata.java
index 049d9d0327..1cd123df14 100644
---
a/tika-serialization/src/main/java/org/apache/tika/serialization/JsonMetadata.java
+++
b/tika-serialization/src/main/java/org/apache/tika/serialization/JsonMetadata.java
@@ -44,6 +44,7 @@ public class JsonMetadata {
.build();
private static volatile StreamReadConstraints streamReadConstraints =
DEFAULT_CONSTRAINTS;
+ private static volatile boolean typedValues = false;
private static volatile ObjectMapper OBJECT_MAPPER;
private static volatile ObjectMapper PRETTY_SERIALIZER;
@@ -58,12 +59,18 @@ public class JsonMetadata {
// Use TikaModule which includes Metadata serializers
ObjectMapper mapper = new ObjectMapper(factory);
mapper.registerModule(new TikaModule());
+ StreamReadConstraints valueConstraints = typedValueConstraints();
+ if (valueConstraints != null) {
+ // registered after TikaModule, so this serializer wins
+ mapper.registerModule(new SimpleModule("TypedMetadata")
+ .addSerializer(Metadata.class, new
MetadataSerializer(false, valueConstraints)));
+ }
OBJECT_MAPPER = mapper;
// Pretty printer needs custom serializer with sort flag
ObjectMapper prettyMapper = new ObjectMapper(factory);
SimpleModule prettySerializerModule = new SimpleModule();
- prettySerializerModule.addSerializer(Metadata.class, new
MetadataSerializer(true));
+ prettySerializerModule.addSerializer(Metadata.class, new
MetadataSerializer(true, valueConstraints));
prettyMapper.registerModule(prettySerializerModule);
PRETTY_SERIALIZER = prettyMapper;
}
@@ -82,6 +89,35 @@ public class JsonMetadata {
rebuildObjectMappers();
}
+ /**
+ * The constraints a value nested in a metadata object must meet so that
{@link #fromJson(Reader)}
+ * can read the whole back; null when values are written as strings.
+ */
+ private static StreamReadConstraints typedValueConstraints() {
+ if (!typedValues) {
+ return null;
+ }
+ return streamReadConstraints.rebuild()
+ .maxNestingDepth(Math.max(0,
streamReadConstraints.getMaxNestingDepth() - 1))
+ .build();
+ }
+
+ /**
+ * Opt-in: write values of registered INTEGER, REAL and BOOLEAN keys as
JSON numbers and
+ * booleans, and of JSON keys (e.g. {@code tk:chunks}) as JSON rather than
an escaped string.
+ * Off by default for backwards compatibility. {@link #fromJson(Reader)}
reads either form.
+ * <p>
+ * Typically called by TikaLoader from the "metadata-list" configuration
section.
+ */
+ public static synchronized void setTypedValues(boolean typed) {
+ typedValues = typed;
+ rebuildObjectMappers();
+ }
+
+ public static boolean isTypedValues() {
+ return typedValues;
+ }
+
/**
* Gets the current stream read constraints.
*
diff --git
a/tika-serialization/src/main/java/org/apache/tika/serialization/JsonMetadataList.java
b/tika-serialization/src/main/java/org/apache/tika/serialization/JsonMetadataList.java
index 03d6b53cc0..206a1e0423 100644
---
a/tika-serialization/src/main/java/org/apache/tika/serialization/JsonMetadataList.java
+++
b/tika-serialization/src/main/java/org/apache/tika/serialization/JsonMetadataList.java
@@ -45,6 +45,7 @@ public class JsonMetadataList {
.build();
private static volatile StreamReadConstraints streamReadConstraints =
DEFAULT_CONSTRAINTS;
+ private static volatile boolean typedValues = false;
private static volatile ObjectMapper OBJECT_MAPPER;
private static volatile ObjectMapper PRETTY_SERIALIZER;
@@ -59,12 +60,18 @@ public class JsonMetadataList {
// Use TikaModule which includes Metadata serializers
ObjectMapper mapper = new ObjectMapper(factory);
mapper.registerModule(new TikaModule());
+ StreamReadConstraints valueConstraints = typedValueConstraints();
+ if (valueConstraints != null) {
+ // registered after TikaModule, so this serializer wins
+ mapper.registerModule(new SimpleModule("TypedMetadata")
+ .addSerializer(Metadata.class, new
MetadataSerializer(false, valueConstraints)));
+ }
OBJECT_MAPPER = mapper;
// Pretty printer needs custom serializer with sort flag
ObjectMapper prettyMapper = new ObjectMapper(factory);
SimpleModule prettySerializerModule = new SimpleModule();
- prettySerializerModule.addSerializer(Metadata.class, new
MetadataSerializer(true));
+ prettySerializerModule.addSerializer(Metadata.class, new
MetadataSerializer(true, valueConstraints));
prettyMapper.registerModule(prettySerializerModule);
PRETTY_SERIALIZER = prettyMapper;
}
@@ -83,6 +90,35 @@ public class JsonMetadataList {
rebuildObjectMappers();
}
+ /**
+ * The constraints a value nested in a list of metadata must meet so that
{@link #fromJson(Reader)}
+ * can read the whole back; null when values are written as strings.
+ */
+ private static StreamReadConstraints typedValueConstraints() {
+ if (!typedValues) {
+ return null;
+ }
+ return streamReadConstraints.rebuild()
+ .maxNestingDepth(Math.max(0,
streamReadConstraints.getMaxNestingDepth() - 2))
+ .build();
+ }
+
+ /**
+ * Opt-in: write values of registered INTEGER, REAL and BOOLEAN keys as
JSON numbers and
+ * booleans, and of JSON keys (e.g. {@code tk:chunks}) as JSON rather than
an escaped string.
+ * Off by default for backwards compatibility. {@link #fromJson(Reader)}
reads either form.
+ * <p>
+ * Typically called by TikaLoader from the "metadata-list" configuration
section.
+ */
+ public static synchronized void setTypedValues(boolean typed) {
+ typedValues = typed;
+ rebuildObjectMappers();
+ }
+
+ public static boolean isTypedValues() {
+ return typedValues;
+ }
+
/**
* Gets the current stream read constraints.
*
diff --git
a/tika-serialization/src/main/java/org/apache/tika/serialization/MetadataValueTypes.java
b/tika-serialization/src/main/java/org/apache/tika/serialization/MetadataValueTypes.java
new file mode 100644
index 0000000000..a3881849af
--- /dev/null
+++
b/tika-serialization/src/main/java/org/apache/tika/serialization/MetadataValueTypes.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.serialization;
+
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.Property;
+
+/**
+ * Value types of metadata keys for typed JSON output.
+ * <p>
+ * The {@link Property} registry knows a key only once its declaring class has
loaded in this JVM.
+ * A pipes parent (tika-server's {@code /rmeta}, for one) serializes metadata
from parsers and
+ * plugins it never loads, so the fork reports the types of the keys it wrote
+ * ({@link #describe}) and the parent records them ({@link #learn}) before
serializing.
+ */
+public final class MetadataValueTypes {
+
+ // bounds what a misbehaving fork can add
+ static final int MAX_LEARNED = 10_000;
+
+ private static final Set<Property.ValueType> TYPED =
EnumSet.of(Property.ValueType.BOOLEAN,
+ Property.ValueType.INTEGER, Property.ValueType.REAL,
Property.ValueType.JSON);
+
+ private static final Map<String, Property.ValueType> LEARNED = new
ConcurrentHashMap<>();
+
+ private MetadataValueTypes() {
+ }
+
+ /**
+ * @return the key's type from its {@link Property} declaration in this
JVM, else as reported
+ * by a fork, else null
+ */
+ public static Property.ValueType get(String key) {
+ Property property = Property.get(key);
+ return property != null ? property.getValueType() : LEARNED.get(key);
+ }
+
+ /**
+ * @return the keys in the list that typed output writes as other than
strings, with their
+ * types
+ */
+ public static Map<String, Property.ValueType> describe(List<Metadata>
metadataList) {
+ Map<String, Property.ValueType> types = new TreeMap<>();
+ for (Metadata metadata : metadataList) {
+ for (String name : metadata.names()) {
+ Property.ValueType type = get(name);
+ if (type != null && TYPED.contains(type)) {
+ types.put(name, type);
+ }
+ }
+ }
+ return types;
+ }
+
+ /**
+ * Records a type reported by a fork. Ignored when this JVM declares the
key itself, the key
+ * was already learned, the type name is unknown or written as a string,
or the cap is reached.
+ */
+ public static void learn(String key, String typeName) {
+ if (Property.get(key) != null || LEARNED.size() >= MAX_LEARNED) {
+ return;
+ }
+ Property.ValueType type;
+ try {
+ type = Property.ValueType.valueOf(typeName);
+ } catch (IllegalArgumentException e) {
+ return;
+ }
+ if (TYPED.contains(type)) {
+ LEARNED.putIfAbsent(key, type);
+ }
+ }
+}
diff --git
a/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/MetadataDeserializer.java
b/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/MetadataDeserializer.java
index 248783f97f..0aef15de59 100644
---
a/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/MetadataDeserializer.java
+++
b/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/MetadataDeserializer.java
@@ -17,28 +17,40 @@
package org.apache.tika.serialization.serdes;
import java.io.IOException;
+import java.io.StringWriter;
import com.fasterxml.jackson.core.JacksonException;
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.util.TokenBuffer;
import org.apache.tika.metadata.Metadata;
+/**
+ * Reads what {@link MetadataSerializer} writes, typed or not: a scalar is one
value, an array of
+ * scalars is several, and an object, or an array holding an object or array,
is one JSON value.
+ * Needs no knowledge of the keys' types.
+ */
public class MetadataDeserializer extends JsonDeserializer<Metadata> {
+ private static final JsonFactory JSON_FACTORY = new JsonFactory();
+
@Override
public Metadata deserialize(JsonParser jsonParser, DeserializationContext
deserializationContext) throws IOException, JacksonException {
Metadata metadata = new Metadata();
boolean keepGoing = true;
while (keepGoing) {
- keepGoing = addField(jsonParser, metadata);
+ keepGoing = addField(jsonParser, deserializationContext, metadata);
}
return metadata;
}
- private boolean addField(JsonParser jsonParser, Metadata metadata) throws
IOException {
+ private boolean addField(JsonParser jsonParser, DeserializationContext
ctxt, Metadata metadata)
+ throws IOException {
String field = jsonParser.nextFieldName();
if (field == null) {
return false;
@@ -51,21 +63,47 @@ public class MetadataDeserializer extends
JsonDeserializer<Metadata> {
if (token.isScalarValue()) {
metadata.reconstruct(field, jsonParser.getText(), false);
- } else if (jsonParser.isExpectedStartArrayToken()) {
- token = jsonParser.nextToken();
- while (token != null) {
- if (token == JsonToken.END_ARRAY) {
+ return true;
+ }
+ if (!token.isStructStart()) {
+ return false;
+ }
+ TokenBuffer buffer = new TokenBuffer(jsonParser, ctxt);
+ buffer.copyCurrentStructure(jsonParser);
+ if (isOneValue(buffer)) {
+ metadata.reconstruct(field, toJson(buffer), false);
+ return true;
+ }
+ try (JsonParser values = buffer.asParser()) {
+ values.nextToken();
+ while (values.nextToken() != JsonToken.END_ARRAY) {
+ metadata.reconstruct(field, values.getText(), true);
+ }
+ }
+ return true;
+ }
+
+ /** True for an object, or an array holding an object or array; false for
an array of scalars. */
+ static boolean isOneValue(TokenBuffer buffer) throws IOException {
+ try (JsonParser parser = buffer.asParser()) {
+ if (parser.nextToken() != JsonToken.START_ARRAY) {
+ return parser.currentToken() == JsonToken.START_OBJECT;
+ }
+ JsonToken token;
+ while ((token = parser.nextToken()) != JsonToken.END_ARRAY) {
+ if (token.isStructStart()) {
return true;
- } else if (token.isScalarValue()) {
- metadata.reconstruct(field, jsonParser.getText(), true);
- } else {
- break;
}
- token = jsonParser.nextToken();
}
- } else {
return false;
}
- return true;
+ }
+
+ private static String toJson(TokenBuffer buffer) throws IOException {
+ StringWriter writer = new StringWriter();
+ try (JsonGenerator generator = JSON_FACTORY.createGenerator(writer)) {
+ buffer.serialize(generator);
+ }
+ return writer.toString();
}
}
diff --git
a/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/MetadataSerializer.java
b/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/MetadataSerializer.java
index db67f6e51a..5487192625 100644
---
a/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/MetadataSerializer.java
+++
b/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/MetadataSerializer.java
@@ -19,17 +19,29 @@ package org.apache.tika.serialization.serdes;
import java.io.IOException;
import java.util.Arrays;
import java.util.Comparator;
+import java.util.regex.Pattern;
+import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.StreamReadConstraints;
import com.fasterxml.jackson.databind.JsonSerializer;
import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.util.TokenBuffer;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.Property;
import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.serialization.MetadataValueTypes;
public class MetadataSerializer extends JsonSerializer<Metadata> {
private static final String TIKA_CONTENT_KEY =
TikaCoreProperties.TIKA_CONTENT.getName();
+ private static final Pattern JSON_INTEGER =
Pattern.compile("-?(?:0|[1-9][0-9]*)");
+ private static final Pattern JSON_NUMBER =
+
Pattern.compile("-?(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?(?:[eE][+-]?[0-9]+)?");
+
//always sort the content at the end
private static final Comparator<String> METADATA_KEY_COMPARATOR = new
Comparator<String>() {
@Override
@@ -46,13 +58,35 @@ public class MetadataSerializer extends
JsonSerializer<Metadata> {
private boolean prettyPrint = false;
- public MetadataSerializer() {
+ // null: every value is written as a string
+ private final JsonFactory typedValueFactory;
+ public MetadataSerializer() {
+ this(false);
}
public MetadataSerializer(boolean prettyPrint) {
+ this(prettyPrint, null);
+ }
+
+ /**
+ * @param typedValueConstraints if non-null, values of registered {@link
Property.ValueType#INTEGER},
+ * {@link Property.ValueType#REAL}, {@link
Property.ValueType#BOOLEAN}
+ * and {@link Property.ValueType#JSON} keys
are written as JSON numbers,
+ * booleans and structures. A value that is
malformed for its type, or
+ * that a reader held to these constraints
could not read back, stays
+ * a string.
+ */
+ public MetadataSerializer(boolean prettyPrint, StreamReadConstraints
typedValueConstraints) {
this.prettyPrint = prettyPrint;
+ if (typedValueConstraints == null) {
+ this.typedValueFactory = null;
+ } else {
+ this.typedValueFactory = new JsonFactory();
+
this.typedValueFactory.setStreamReadConstraints(typedValueConstraints);
+ }
}
+
@Override
public void serialize(Metadata metadata, JsonGenerator jsonGenerator,
SerializerProvider serializerProvider) throws IOException {
jsonGenerator.writeStartObject();
@@ -64,14 +98,72 @@ public class MetadataSerializer extends
JsonSerializer<Metadata> {
String[] v = metadata.getValues(n);
if (v.length == 0) {
continue;
- } else if (v.length == 1) {
- jsonGenerator.writeStringField(n, v[0]);
+ }
+ Property.ValueType type = valueType(n);
+ jsonGenerator.writeFieldName(n);
+ if (v.length == 1) {
+ if (type != Property.ValueType.JSON || !writeJson(v[0],
jsonGenerator)) {
+ writeValue(type, v[0], jsonGenerator);
+ }
} else {
- jsonGenerator.writeFieldName(n);
- jsonGenerator.writeArray(v, 0, v.length);
+ jsonGenerator.writeStartArray();
+ for (String s : v) {
+ writeValue(type, s, jsonGenerator);
+ }
+ jsonGenerator.writeEndArray();
}
}
jsonGenerator.writeEndObject();
}
+ private Property.ValueType valueType(String name) {
+ if (typedValueFactory == null) {
+ return null;
+ }
+ return MetadataValueTypes.get(name);
+ }
+
+ private void writeValue(Property.ValueType type, String value,
JsonGenerator jsonGenerator)
+ throws IOException {
+ if (type == Property.ValueType.BOOLEAN && ("true".equals(value) ||
"false".equals(value))) {
+ jsonGenerator.writeBoolean(Boolean.parseBoolean(value));
+ } else if ((type == Property.ValueType.INTEGER &&
isNumber(JSON_INTEGER, value))
+ || (type == Property.ValueType.REAL && isNumber(JSON_NUMBER,
value))) {
+ jsonGenerator.writeNumber(value);
+ } else {
+ jsonGenerator.writeString(value);
+ }
+ }
+
+ private boolean isNumber(Pattern pattern, String value) {
+ return value.length() <=
typedValueFactory.streamReadConstraints().getMaxNumberLength()
+ && pattern.matcher(value).matches();
+ }
+
+ /**
+ * Writes the value as JSON if it is one JSON document that {@link
MetadataDeserializer}
+ * reads back as this one value, not as several.
+ */
+ private boolean writeJson(String value, JsonGenerator jsonGenerator)
throws IOException {
+ TokenBuffer buffer;
+ try (JsonParser parser = typedValueFactory.createParser(value)) {
+ if (parser.nextToken() == null) {
+ return false;
+ }
+ buffer = new TokenBuffer(parser);
+ buffer.copyCurrentStructure(parser);
+ if (parser.nextToken() != null) {
+ return false;
+ }
+ } catch (JsonProcessingException e) {
+ return false;
+ }
+ if (!MetadataDeserializer.isOneValue(buffer)) {
+ return false;
+ }
+ // the buffer, not writeRawValue: keeps number text exact and honors
pretty printing
+ buffer.serialize(jsonGenerator);
+ return true;
+ }
+
}
diff --git
a/tika-serialization/src/test/java/org/apache/tika/serialization/MetadataValueTypesTest.java
b/tika-serialization/src/test/java/org/apache/tika/serialization/MetadataValueTypesTest.java
new file mode 100644
index 0000000000..ab49ded299
--- /dev/null
+++
b/tika-serialization/src/test/java/org/apache/tika/serialization/MetadataValueTypesTest.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.serialization;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+import java.io.StringWriter;
+import java.util.List;
+import java.util.Map;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+
+import org.apache.tika.metadata.Audio;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.Property;
+import org.apache.tika.metadata.TikaCoreProperties;
+
+public class MetadataValueTypesTest {
+
+ // keys are never declared, so only learn() can type them; each test uses
its own
+ private static final String PREFIX = "test-value-types-";
+
+ @AfterEach
+ public void reset() {
+ JsonMetadataList.setTypedValues(false);
+ }
+
+ @Test
+ public void testLearnedTypeIsWrittenTyped() throws Exception {
+ String key = PREFIX + "learned:count";
+ Metadata m = new Metadata();
+ m.set(key, "7");
+ JsonMetadataList.setTypedValues(true);
+ assertEquals("7", toJson(m).get(key).textValue());
+
+ MetadataValueTypes.learn(key, "INTEGER");
+ assertEquals(7, toJson(m).get(key).intValue());
+ }
+
+ @Test
+ public void testLocalDeclarationWins() {
+ String title = TikaCoreProperties.TITLE.getName();
+ MetadataValueTypes.learn(title, "INTEGER");
+ assertEquals(Property.ValueType.TEXT, MetadataValueTypes.get(title));
+ }
+
+ @Test
+ public void testFirstLearnedWins() {
+ String key = PREFIX + "first:flag";
+ MetadataValueTypes.learn(key, "BOOLEAN");
+ MetadataValueTypes.learn(key, "INTEGER");
+ assertEquals(Property.ValueType.BOOLEAN, MetadataValueTypes.get(key));
+ }
+
+ @Test
+ public void testUntypedAndUnknownTypesIgnored() {
+ for (String type : new String[]{"TEXT", "DATE", "nope", ""}) {
+ String key = PREFIX + "ignored:" + type;
+ MetadataValueTypes.learn(key, type);
+ assertNull(MetadataValueTypes.get(key), type);
+ }
+ }
+
+ @Test
+ public void testDescribeReportsOnlyTypedKeys() {
+ Metadata m = new Metadata();
+ m.set(Audio.CHANNELS, "2");
+ m.set(TikaCoreProperties.TITLE, "t");
+ m.set(PREFIX + "undeclared", "3");
+ assertEquals(Map.of(Audio.CHANNELS.getName(),
Property.ValueType.INTEGER),
+ MetadataValueTypes.describe(List.of(m)));
+ }
+
+ private static JsonNode toJson(Metadata m) throws Exception {
+ StringWriter writer = new StringWriter();
+ JsonMetadataList.toJson(List.of(m), writer);
+ return new ObjectMapper().readTree(writer.toString()).get(0);
+ }
+}
diff --git
a/tika-serialization/src/test/java/org/apache/tika/serialization/TypedMetadataValuesTest.java
b/tika-serialization/src/test/java/org/apache/tika/serialization/TypedMetadataValuesTest.java
new file mode 100644
index 0000000000..fc159fd123
--- /dev/null
+++
b/tika-serialization/src/test/java/org/apache/tika/serialization/TypedMetadataValuesTest.java
@@ -0,0 +1,272 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.serialization;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Random;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import org.apache.tika.config.loader.TikaLoader;
+import org.apache.tika.exception.TikaConfigException;
+import org.apache.tika.metadata.Audio;
+import org.apache.tika.metadata.Geographic;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.Property;
+import org.apache.tika.metadata.TikaCoreProperties;
+
+public class TypedMetadataValuesTest {
+
+ private static final String CHUNKS =
TikaCoreProperties.TIKA_CHUNKS.getName();
+ private static final Property TEST_JSON =
Property.internalJson("test-typed:json");
+ // number text a double would rewrite
+ private static final String CHUNKS_JSON =
+ "[{\"text\":\"a
\\\"b\\\"\",\"x\":1.50,\"y\":1e-5,\"z\":12345678901234567890123," +
+ "\"locators\":{\"text\":[{\"start\":0,\"end\":5}]}}]";
+
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
+ @TempDir
+ Path tmp;
+
+ @AfterEach
+ public void reset() {
+ setTyped(false);
+ }
+
+ @Test
+ public void testOffByDefault() throws Exception {
+ Metadata m = chunks(CHUNKS_JSON);
+ m.set(Audio.CHANNELS, "2");
+ JsonNode node = MAPPER.readTree(toJsonList(m)).get(0);
+ assertEquals(CHUNKS_JSON, node.get(CHUNKS).textValue());
+ assertEquals("2", node.get(Audio.CHANNELS.getName()).textValue());
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ public void testChunksAreJson(boolean pretty) throws Exception {
+ setTyped(true);
+ Metadata m = chunks(CHUNKS_JSON);
+ String json = toJsonList(m, pretty);
+ JsonNode chunks = MAPPER.readTree(json).get(0).get(CHUNKS);
+ assertTrue(chunks.isArray(), json);
+ assertEquals("a \"b\"", chunks.get(0).get("text").textValue());
+ assertTrue(json.contains("1.50") && json.contains("1e-5"), "number
text kept: " + json);
+
+ Metadata back = JsonMetadataList.fromJson(new
StringReader(json)).get(0);
+ assertEquals(CHUNKS_JSON, back.get(CHUNKS));
+ }
+
+ @Test
+ public void testSingleMetadataChunksAreJson() throws Exception {
+ setTyped(true);
+ StringWriter writer = new StringWriter();
+ JsonMetadata.toJson(chunks(CHUNKS_JSON), writer);
+ assertTrue(MAPPER.readTree(writer.toString()).get(CHUNKS).isArray());
+ assertEquals(CHUNKS_JSON,
+ JsonMetadata.fromJson(new
StringReader(writer.toString())).get(CHUNKS));
+ }
+
+ @Test
+ public void testNumbersAndBooleans() throws Exception {
+ setTyped(true);
+ Metadata m = new Metadata();
+ m.set(Audio.CHANNELS, "2");
+ m.set(Geographic.LATITUDE, "-12.50");
+ m.set(Audio.HAS_DRM, "true");
+ JsonNode node = MAPPER.readTree(toJsonList(m)).get(0);
+ assertTrue(node.get(Audio.CHANNELS.getName()).isIntegralNumber());
+ assertTrue(node.get(Geographic.LATITUDE.getName()).isNumber());
+ assertTrue(node.get(Audio.HAS_DRM.getName()).isBoolean());
+ assertRoundTrips(m);
+ }
+
+ /** A value malformed for its type stays a string rather than becoming
invalid JSON. */
+ @ParameterizedTest
+ @ValueSource(strings = {"abc", "0042", "1.", "+1", "NaN", "Infinity",
"1,5", " 1", ""})
+ public void testMalformedNumbersStayStrings(String value) throws Exception
{
+ setTyped(true);
+ Metadata m = new Metadata();
+ m.set(Audio.CHANNELS, value);
+ m.set(Geographic.LATITUDE, value);
+ m.set(Audio.HAS_DRM, value);
+ JsonNode node = MAPPER.readTree(toJsonList(m)).get(0);
+ assertTrue(node.get(Audio.CHANNELS.getName()).isTextual());
+ assertTrue(node.get(Geographic.LATITUDE.getName()).isTextual());
+ assertTrue(node.get(Audio.HAS_DRM.getName()).isTextual());
+ assertRoundTrips(m);
+ }
+
+ /**
+ * Each would read back as something other than the one value written: a
scalar or an
+ * array of scalars looks like an ordinary value or values; the rest are
not one document.
+ */
+ @ParameterizedTest
+ @ValueSource(strings = {"[\"a\",\"b\"]", "[]", "[1,2]", "\"a\"", "5",
"null", "{\"a\":",
+ "{} {}", "not json", ""})
+ public void testJsonThatWouldNotRoundTripStaysAString(String value) throws
Exception {
+ setTyped(true);
+ JsonNode node = MAPPER.readTree(toJsonList(chunks(value))).get(0);
+ assertTrue(node.get(CHUNKS).isTextual(), value);
+ assertRoundTrips(chunks(value));
+ }
+
+ @Test
+ public void testMultipleJsonValuesStayStrings() throws Exception {
+ setTyped(true);
+ Metadata m = new Metadata();
+ m.add(TEST_JSON.getName(), CHUNKS_JSON);
+ m.add(TEST_JSON.getName(), "{\"a\":1}");
+ JsonNode node =
MAPPER.readTree(toJsonList(m)).get(0).get(TEST_JSON.getName());
+ assertTrue(node.isArray() && node.get(0).isTextual() &&
node.get(1).isTextual());
+ assertRoundTrips(m);
+ }
+
+ /** Deeper than the reader's nesting limit allows stays a string, so
fromJson still works. */
+ @Test
+ public void testTooDeepStaysAString() throws Exception {
+ setTyped(true);
+ int max =
JsonMetadataList.getStreamReadConstraints().getMaxNestingDepth();
+ // list + metadata object leave max - 2 for the value
+ String fits = "[".repeat(max - 3) + "{}" + "]".repeat(max - 3);
+ String tooDeep = "[" + fits + "]";
+
assertTrue(MAPPER.readTree(toJsonList(chunks(fits))).get(0).get(CHUNKS).isArray());
+
assertTrue(MAPPER.readTree(toJsonList(chunks(tooDeep))).get(0).get(CHUNKS).isTextual());
+ assertRoundTrips(chunks(fits));
+ assertRoundTrips(chunks(tooDeep));
+ }
+
+ /** Typed output from a writer reads back the same whatever the keys and
values. */
+ @Test
+ public void testRandomRoundTrip() throws Exception {
+ long seed = System.nanoTime();
+ Random random = new Random(seed);
+ String[] keys = {TEST_JSON.getName(), Audio.CHANNELS.getName(),
Geographic.LATITUDE.getName(),
+ Audio.HAS_DRM.getName(), "custom:key"};
+ String[] values = {CHUNKS_JSON, "{\"a\":[1,{\"b\":null}]}", "[[1],2]",
"[\"x\"]", "[]",
+ "{}", "0", "-0.5e+3", "007", "true", "false", "TRUE", "", "a",
"null", "\"q\"",
+ "1.0E-5"};
+ for (boolean typed : new boolean[]{false, true}) {
+ setTyped(typed);
+ for (int i = 0; i < 200; i++) {
+ Metadata m = random.nextBoolean() ?
chunks(values[random.nextInt(values.length)])
+ : new Metadata();
+ for (String key : keys) {
+ int n = random.nextInt(3);
+ for (int j = 0; j < n; j++) {
+ m.add(key, values[random.nextInt(values.length)]);
+ }
+ }
+ assertRoundTrips(m, "seed=" + seed);
+ }
+ }
+ }
+
+ @Test
+ public void testConfig() throws Exception {
+ int depth =
JsonMetadataList.getStreamReadConstraints().getMaxNestingDepth();
+ TikaLoader.load(config("{\"metadata-list\": {\"typedValues\":
true}}"));
+ assertTrue(JsonMetadataList.isTypedValues());
+ assertTrue(JsonMetadata.isTypedValues());
+ // typedValues alone leaves the read limits as they were
+ assertEquals(depth,
JsonMetadataList.getStreamReadConstraints().getMaxNestingDepth());
+
+ TikaLoader.load(config("{\"metadata-list\": {\"typedValues\":
false}}"));
+ assertFalse(JsonMetadataList.isTypedValues());
+ assertFalse(JsonMetadata.isTypedValues());
+
+ assertThrows(TikaConfigException.class,
+ () -> TikaLoader.load(config("{\"metadata-list\":
{\"typedValues\": \"yes\"}}")));
+ }
+
+ private Path config(String json) throws Exception {
+ Path p = Files.createTempFile(tmp, "tika-config", ".json");
+ Files.writeString(p, json, StandardCharsets.UTF_8);
+ return p;
+ }
+
+ private static void setTyped(boolean typed) {
+ JsonMetadata.setTypedValues(typed);
+ JsonMetadataList.setTypedValues(typed);
+ }
+
+ private static Metadata chunks(String value) {
+ Metadata m = new Metadata();
+ m.set(TikaCoreProperties.TIKA_CHUNKS, value);
+ return m;
+ }
+
+ private static String toJsonList(Metadata m) throws Exception {
+ return toJsonList(m, false);
+ }
+
+ private static String toJsonList(Metadata m, boolean pretty) throws
Exception {
+ StringWriter writer = new StringWriter();
+ JsonMetadataList.toJson(List.of(m), writer, pretty);
+ return writer.toString();
+ }
+
+ private static void assertRoundTrips(Metadata m) throws Exception {
+ assertRoundTrips(m, "");
+ }
+
+ private static void assertRoundTrips(Metadata m, String message) throws
Exception {
+ for (boolean pretty : new boolean[]{false, true}) {
+ String json = toJsonList(m, pretty);
+ Metadata back = JsonMetadataList.fromJson(new
StringReader(json)).get(0);
+ assertArrayEquals(m.names().length == 0 ? new String[0] :
sorted(m.names()),
+ sorted(back.names()), message + " " + json);
+ for (String name : m.names()) {
+ assertArrayEquals(m.getValues(name), back.getValues(name),
+ message + " " + name + " " + json);
+ }
+
+ StringWriter writer = new StringWriter();
+ JsonMetadata.toJson(m, writer);
+ Metadata single = JsonMetadata.fromJson(new
StringReader(writer.toString()));
+ for (String name : m.names()) {
+ assertArrayEquals(m.getValues(name), single.getValues(name),
+ message + " " + name + " " + writer);
+ }
+ }
+ }
+
+ private static String[] sorted(String[] names) {
+ String[] copy = names.clone();
+ Arrays.sort(copy);
+ return copy;
+ }
+}
