This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 532a685ea1 TIKA-4808 - remove access to the network parser from cli
(#3036)
532a685ea1 is described below
commit 532a685ea1bd4db96befd17447f3a9aec70bca54
Author: Tim Allison <[email protected]>
AuthorDate: Mon Aug 17 20:44:42 2026 -0400
TIKA-4808 - remove access to the network parser from cli (#3036)
---
CHANGES.txt | 22 ++-
docs/modules/ROOT/pages/using-tika/cli/index.adoc | 12 +-
.../src/main/java/org/apache/tika/cli/TikaCLI.java | 20 +--
.../java/org/apache/tika/parser/NetworkParser.java | 188 ---------------------
.../org/apache/tika/parser/NetworkParserTest.java | 135 ---------------
.../tika/metadata/metadata-open-namespaces.json | 1 -
6 files changed, 30 insertions(+), 348 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 558f6d9b81..c8f38747dc 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -2,14 +2,20 @@ Release 4.0.0 - ???
BREAKING CHANGES
- * tika-app: the inline short forms -eX (output encoding), -pX (document
- password) and -c<uri> (network client) were removed from standard mode.
- Use --encoding=X, --password=X and --client=<uri>. These were the only
- short flags that consumed an inline value, and matching them by prefix
- meant a long name written with one dash was silently swallowed:
- -config=tika.json set the network-client URI to "onfig=tika.json" and
- loaded no config file, with no error. Every single-dash long name is now
- rejected with a message naming the two-dash form (TIKA-4808).
+ * tika-app: the inline short forms -eX (output encoding) and -pX (document
+ password) were removed from standard mode. Use --encoding=X and
+ --password=X. These were the only short flags that consumed an inline
+ value, and matching them by prefix meant a long name written with one
+ dash was silently swallowed: -config=tika.json set the password to
+ "onfig=tika.json" and loaded no config file, with no error. Every
+ single-dash long name is now rejected with a message naming the
+ two-dash form (TIKA-4808).
+
+ * tika-core/tika-app: NetworkParser and tika-app's -c/--client=<uri>
+ network-client mode were removed. It dispatched raw sockets
+ (including a bare telnet:// scheme) to an arbitrary user-supplied
+ host with no auth or TLS enforcement; use tika-server instead
+ (TIKA-4808).
* Metadata's reserved tk: (and legacy X-TIKA:) namespace is now a trust
boundary for String-keyed writes. A String write to a reserved key throws
diff --git a/docs/modules/ROOT/pages/using-tika/cli/index.adoc
b/docs/modules/ROOT/pages/using-tika/cli/index.adoc
index cef8b5849b..121a1e3e12 100644
--- a/docs/modules/ROOT/pages/using-tika/cli/index.adoc
+++ b/docs/modules/ROOT/pages/using-tika/cli/index.adoc
@@ -64,10 +64,14 @@ Writing a long name with a single dash is always an error —
`-input`, `-config
`-encoding=UTF-8` and the like are rejected with a message naming the two-dash
form. No short
flag takes an inline value, so nothing is silently consumed as a flag's
argument.
-NOTE: In 4.0.0 the inline short forms `-eX` (encoding), `-pX` (password) and
`-c<uri>`
-(network client) were removed from standard mode. Use `--encoding=X`,
`--password=X` and
-`--client=<uri>`. Scripts using the old forms now fail with an
unrecognized-option error
-rather than silently misparsing.
+NOTE: In 4.0.0 the inline short forms `-eX` (encoding) and `-pX` (password)
were removed
+from standard mode. Use `--encoding=X` and `--password=X`. Scripts using the
old forms now
+fail with an unrecognized-option error rather than silently misparsing.
+
+NOTE: In 4.0.0 network-client mode (`-c<uri>` / `--client=<uri>`, backed by
+`NetworkParser`) was removed entirely. It dispatched raw sockets -- including
a bare
+`telnet://` scheme -- to an arbitrary user-supplied host with no
authentication or TLS
+enforcement. Use tika-server instead.
`-T` means different things in the two modes, so read it in context:
`--text-main` in standard
mode, `--timeoutMillis=<ms>` in pipes mode. `-p` (`--pluginsDir`) and `-c`
(`--config`) exist
diff --git a/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java
b/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java
index c6a31730e7..1089229760 100644
--- a/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java
+++ b/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java
@@ -32,7 +32,6 @@ import java.io.Serializable;
import java.io.UnsupportedEncodingException;
import java.io.Writer;
import java.lang.reflect.Field;
-import java.net.URI;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.InvalidPathException;
@@ -91,7 +90,6 @@ import org.apache.tika.mime.MimeTypeException;
import org.apache.tika.mime.MimeTypes;
import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.parser.CompositeParser;
-import org.apache.tika.parser.NetworkParser;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.parser.ParserDecorator;
@@ -136,7 +134,6 @@ public class TikaCLI {
private TikaLoader tikaLoader;
private String configFilePath;
private boolean recursiveJSON = false;
- private URI networkURI = null;
/**
* Output character encoding, or <code>null</code> for platform default
*/
@@ -447,8 +444,13 @@ public class TikaCLI {
}
if (args.length == 2) {
- if (Files.isDirectory(Paths.get(args[0]))) {
- return true;
+ try {
+ if (Files.isDirectory(Paths.get(args[0]))) {
+ return true;
+ }
+ } catch (InvalidPathException e) {
+ // Not a valid path (e.g. a URL passed as a raw single-dash
+ // arg on Windows) -- fall through to the other checks.
}
}
@@ -607,8 +609,6 @@ public class TikaCLI {
maxEmbeddedCount =
Integer.parseInt(arg.substring("--maxEmbeddedCount=".length()));
} else if (arg.equals("-r") || arg.equals("--pretty-print")) {
prettyPrint = true;
- } else if (arg.startsWith("--client=")) {
- networkURI = new URI(arg.substring("--client=".length()));
} else {
// Any arg that reaches here is either "-" (stdin), an existing
// file, a URL, or an unknown/typo'd flag. The default fallthrough
@@ -1023,11 +1023,7 @@ public class TikaCLI {
Files.deleteIfExists(tempConfig);
}
}
- if (networkURI != null) {
- parser = new NetworkParser(networkURI);
- } else {
- parser = tikaLoader.loadAutoDetectParser();
- }
+ parser = tikaLoader.loadAutoDetectParser();
// Load configs from tika-config.json and merge into existing context
// (preserves EmbeddedDocumentExtractor and other items set before
configure())
diff --git a/tika-core/src/main/java/org/apache/tika/parser/NetworkParser.java
b/tika-core/src/main/java/org/apache/tika/parser/NetworkParser.java
deleted file mode 100644
index f6bae48eef..0000000000
--- a/tika-core/src/main/java/org/apache/tika/parser/NetworkParser.java
+++ /dev/null
@@ -1,188 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.parser;
-
-import java.io.FilterOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.Socket;
-import java.net.URI;
-import java.net.URL;
-import java.net.URLConnection;
-import java.util.Collections;
-import java.util.Set;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.io.function.IOSupplier;
-import org.apache.commons.io.input.CloseShieldInputStream;
-import org.xml.sax.Attributes;
-import org.xml.sax.ContentHandler;
-import org.xml.sax.SAXException;
-import org.xml.sax.helpers.DefaultHandler;
-
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.io.TikaInputStream;
-import org.apache.tika.metadata.KeyPrefix;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.mime.MediaType;
-import org.apache.tika.sax.TaggedContentHandler;
-import org.apache.tika.sax.TeeContentHandler;
-import org.apache.tika.utils.XMLReaderUtils;
-
-
-public class NetworkParser implements Parser {
-
- // meta/@name in the remote parse service's XML response: naming a
service's own inferred
- // field is tool provenance, not file provenance (contrast
dif:/gdal:/hdf:, which read names
- // straight out of the file being parsed).
- private static final KeyPrefix NETWORK =
- KeyPrefix.tool("network:", "meta/@name from a remote parse
service's XML response");
-
- private final URI uri;
-
- private final Set<MediaType> supportedTypes;
-
- public NetworkParser(URI uri, Set<MediaType> supportedTypes) {
- this.uri = uri;
- this.supportedTypes = supportedTypes;
- }
-
- public NetworkParser(URI uri) {
- this(uri, Collections.singleton(MediaType.OCTET_STREAM));
- }
-
- public Set<MediaType> getSupportedTypes(ParseContext context) {
- return supportedTypes;
- }
-
- public void parse(TikaInputStream tis, ContentHandler handler, Metadata
metadata,
- ParseContext context) throws IOException, SAXException,
TikaException {
- if ("telnet".equals(uri.getScheme())) {
- try (Socket socket = new Socket(uri.getHost(), uri.getPort())) {
- new ParsingTask(tis, new
FilterOutputStream(socket.getOutputStream()) {
- @Override
- public void close() throws IOException {
- socket.shutdownOutput();
- }
- }).parse(socket::getInputStream, handler, metadata, context);
- }
- } else {
- URL url = uri.toURL();
- URLConnection connection = url.openConnection();
- connection.setDoOutput(true);
- connection.connect();
- // getInputStream() must not be called until the request is
written (see
- // ParsingTask.parse() javadoc) -- get the output stream first.
- OutputStream output = connection.getOutputStream();
- new ParsingTask(tis, output)
- .parse(connection::getInputStream, handler, metadata,
context);
- }
-
- }
-
- private static class ParsingTask implements Runnable {
-
- private final TikaInputStream input;
-
- private final OutputStream output;
-
- private volatile Exception exception = null;
-
- public ParsingTask(TikaInputStream input, OutputStream output) {
- this.input = input;
- this.output = output;
- }
-
- /**
- * @param streamSupplier opens the response stream -- called only
after the writer
- * thread has started, so a supplier that blocks
until the request
- * is fully sent (e.g. {@code
URLConnection#getInputStream}) doesn't
- * deadlock waiting on a write that hasn't begun
yet
- */
- public void parse(IOSupplier<InputStream> streamSupplier,
ContentHandler handler,
- Metadata metadata, ParseContext context)
- throws IOException, SAXException, TikaException {
- Thread thread = new Thread(this, "Tika network parser");
- thread.start();
-
- TaggedContentHandler tagged =
- new TaggedContentHandler(handler);
- // shield the real stream from the SAX parser's own close(); the
try-with-resources
- // below is what actually closes it, once parsing is done.
- try (InputStream stream = streamSupplier.get()) {
- XMLReaderUtils
- .parseSAX(CloseShieldInputStream.wrap(stream),
- new TeeContentHandler(tagged, new
MetaHandler(metadata)), context);
- } catch (SAXException e) {
- tagged.throwIfCauseOf(e);
- throw new TikaException("Invalid network parser output", e);
- } catch (IOException e) {
- throw new TikaException("Unable to read network parser
output", e);
- } finally {
- try {
- thread.join(1000);
- } catch (InterruptedException e) {
- throw new TikaException("Network parser interrupted", e);
- }
-
- if (exception != null) {
- input.throwIfCauseOf(exception);
- throw new TikaException("Unexpected network parser error",
exception);
- }
- }
- }
-
- //----------------------------------------------------------<Runnable>
-
- public void run() {
- try {
- try {
- IOUtils.copy(input, output);
- } finally {
- output.close();
- }
- } catch (Exception e) {
- exception = e;
- }
- }
-
- }
-
- private static class MetaHandler extends DefaultHandler {
-
- private final Metadata metadata;
-
- public MetaHandler(Metadata metadata) {
- this.metadata = metadata;
- }
-
- @Override
- public void startElement(String uri, String localName, String qName,
Attributes attributes)
- throws SAXException {
- if ("http://www.w3.org/1999/xhtml".equals(uri) &&
"meta".equals(localName)) {
- String name = attributes.getValue("", "name");
- String content = attributes.getValue("", "content");
- if (name != null && content != null) {
- metadata.add(NETWORK, name, content);
- }
- }
- }
-
- }
-
-}
diff --git
a/tika-core/src/test/java/org/apache/tika/parser/NetworkParserTest.java
b/tika-core/src/test/java/org/apache/tika/parser/NetworkParserTest.java
deleted file mode 100644
index a59bfaf7a6..0000000000
--- a/tika-core/src/test/java/org/apache/tika/parser/NetworkParserTest.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.parser;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNull;
-
-import java.io.ByteArrayOutputStream;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.ServerSocket;
-import java.net.Socket;
-import java.net.URI;
-import java.nio.charset.StandardCharsets;
-import java.util.Collections;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.junit.jupiter.api.Test;
-import org.xml.sax.helpers.DefaultHandler;
-
-import org.apache.tika.io.TikaInputStream;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.mime.MediaType;
-
-/**
- * {@code meta/@name} in the remote service's XML response is tool-derived
text, so
- * {@link NetworkParser} routes it through the {@code network:} {@link
- * org.apache.tika.metadata.KeyPrefix}. No test resource fixture is needed or
wanted here: a
- * loopback {@link ServerSocket} stands in for the remote parse service (raw
sockets only --
- * {@code com.sun.net.httpserver} is forbiddenapis-banned as a non-portable
internal JDK class).
- * TIKA-4816.
- */
-public class NetworkParserTest {
-
- private static final String RESPONSE_XML = "<html
xmlns=\"http://www.w3.org/1999/xhtml\";>"
- + "<head><meta name=\"foo\"
content=\"bar\"/></head><body/></html>";
-
- @Test
- public void telnetSchemeRoutesMetaNameThroughNetworkKeyPrefix() throws
Exception {
- try (ServerSocket server = new ServerSocket(0)) {
- Thread serverThread = new Thread(() -> {
- try (Socket accepted = server.accept()) {
- accepted.getInputStream().readAllBytes(); // client
half-closes after writing
-
accepted.getOutputStream().write(RESPONSE_XML.getBytes(StandardCharsets.UTF_8));
- } catch (Exception ignore) {
- // surfaced indirectly: the client-side assertions below
fail instead
- }
- }, "test-network-parser-telnet-server");
- serverThread.start();
-
- URI uri = URI.create("telnet://localhost:" +
server.getLocalPort());
- assertMetaNameRoutedThroughNetworkKeyPrefix(uri);
- serverThread.join(5000);
- }
- }
-
- @Test
- public void httpSchemeRoutesMetaNameThroughNetworkKeyPrefix() throws
Exception {
- // Also exercises the getOutputStream()-before-getInputStream()
ordering fix: the
- // original code called URLConnection#getInputStream() before the
request had been
- // written, which deadlocks/fails against any real HTTP server
(TIKA-4816).
- try (ServerSocket server = new ServerSocket(0)) {
- Thread serverThread = new Thread(() -> {
- try (Socket accepted = server.accept()) {
- // HttpURLConnection keeps the connection open for
keep-alive (never
- // half-closes), so -- unlike the telnet server above --
draining to EOF
- // would hang forever; read exactly the declared request
body instead.
- readHttpRequest(accepted.getInputStream());
- byte[] body =
RESPONSE_XML.getBytes(StandardCharsets.UTF_8);
- OutputStream out = accepted.getOutputStream();
- out.write(("HTTP/1.1 200 OK\r\n"
- + "Content-Type: text/xml; charset=utf-8\r\n"
- + "Content-Length: " + body.length + "\r\n"
- + "Connection:
close\r\n\r\n").getBytes(StandardCharsets.UTF_8));
- out.write(body);
- out.flush();
- } catch (Exception ignore) {
- // surfaced indirectly: the client-side assertions below
fail instead
- }
- }, "test-network-parser-http-server");
- serverThread.start();
-
- URI uri = URI.create("http://localhost:"; + server.getLocalPort() +
"/");
- assertMetaNameRoutedThroughNetworkKeyPrefix(uri);
- serverThread.join(5000);
- }
- }
-
- private static final Pattern CONTENT_LENGTH =
- Pattern.compile("(?i)content-length:\\s*(\\d+)");
-
- /** Reads a minimal HTTP request (headers, then the declared
Content-Length body bytes) off
- * {@code in} without closing it and without needing the client to signal
EOF. */
- private static void readHttpRequest(InputStream in) throws Exception {
- ByteArrayOutputStream headerBytes = new ByteArrayOutputStream();
- int trailingCrLfCrLf = 0; // count of the last 4 bytes matching
"\r\n\r\n" so far
- int[] terminator = {'\r', '\n', '\r', '\n'};
- int b;
- while (trailingCrLfCrLf < terminator.length && (b = in.read()) != -1) {
- headerBytes.write(b);
- trailingCrLfCrLf = (b == terminator[trailingCrLfCrLf]) ?
trailingCrLfCrLf + 1
- : (b == '\r' ? 1 : 0);
- }
- Matcher m =
CONTENT_LENGTH.matcher(headerBytes.toString(StandardCharsets.UTF_8));
- int contentLength = m.find() ? Integer.parseInt(m.group(1)) : 0;
- in.readNBytes(contentLength);
- }
-
- private void assertMetaNameRoutedThroughNetworkKeyPrefix(URI uri) throws
Exception {
- NetworkParser parser = new NetworkParser(uri,
Collections.singleton(MediaType.OCTET_STREAM));
- Metadata metadata = new Metadata();
- try (TikaInputStream tis =
- TikaInputStream.get("posted
document".getBytes(StandardCharsets.UTF_8))) {
- parser.parse(tis, new DefaultHandler(), metadata, new
ParseContext());
- }
-
- assertEquals("bar", metadata.get("network:foo"));
- assertNull(metadata.get("foo"), "the unprefixed legacy key must not
appear");
- }
-}
diff --git
a/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-open-namespaces.json
b/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-open-namespaces.json
index c862db6897..ad7343f2e7 100644
---
a/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-open-namespaces.json
+++
b/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-open-namespaces.json
@@ -29,7 +29,6 @@
{"prefix":"mp4:","provenance":"FILE","description":"QuickTime 'keys' box
item names, read verbatim from file
bytes","module":"tika-parser-audiovideo-module"},
{"prefix":"ner:","provenance":"TOOL","description":"named-entity
types","module":"tika-parser-nlp-module"},
{"prefix":"netcdf:","provenance":"FILE","description":"NetCDF global
attribute names","module":"tika-parser-scientific-module"},
- {"prefix":"network:","provenance":"TOOL","description":"meta/@name from a
remote parse service's XML response","module":"tika-core"},
{"prefix":"numbers:","provenance":"FILE","description":"iWork Numbers
document metadata element names","module":"tika-parser-apple-module"},
{"prefix":"ogg:streams-","provenance":"TOOL","description":"Ogg
per-stream-type/kind counts","module":"tika-parser-audiovideo-module"},
{"prefix":"pages:","provenance":"FILE","description":"Pages metadata element
names with no mapped Property","module":"tika-parser-apple-module"},
