This is an automated email from the ASF dual-hosted git repository. tballison pushed a commit to branch TIKA-4809-stage-9 in repository https://gitbox.apache.org/repos/asf/tika.git
commit 6548ea8a8e6da0c4412f495fc4acce0fe6d29afa Author: tallison <[email protected]> AuthorDate: Tue Aug 11 06:51:45 2026 -0400 TIKA-4809: Make http-fetcher TLS verification reachable, and on by default --- docs/modules/ROOT/pages/pipes/plugins/http.adoc | 50 +++++++++++++++------- .../tika/pipes/fetcher/http/HttpFetcher.java | 23 +++++++--- .../fetcher/http/config/HttpFetcherConfig.java | 18 ++++++++ 3 files changed, 69 insertions(+), 22 deletions(-) diff --git a/docs/modules/ROOT/pages/pipes/plugins/http.adoc b/docs/modules/ROOT/pages/pipes/plugins/http.adoc index 9acab6fbce..74960a32eb 100644 --- a/docs/modules/ROOT/pages/pipes/plugins/http.adoc +++ b/docs/modules/ROOT/pages/pipes/plugins/http.adoc @@ -92,7 +92,11 @@ include::example$pipes-http-fetcher.json[] |`maxRedirects` |`0` -|Maximum number of redirects to follow. `0` means follow none. Not applied to range requests — see <<security-notes>>. +|Maximum number of redirects to follow. `0` means follow none. + +|`verifySsl` +|`true` +|Verify server certificates and hostnames. Set `false` to accept any certificate — see <<security-notes>>. |`maxSpoolSize` |`-1` @@ -139,21 +143,35 @@ endpoint that can reach it (`/pipes`, `/async`). Schemes other than `http`/`https` fail only because no other scheme is registered in the connection manager — that is a side effect of the transport setup, not a check. -* **TLS certificates are not verified, and this is not configurable here.** The - underlying client defaults to `verifySsl=false`, which installs an accept-everything - trust strategy and `NoopHostnameVerifier`. `HttpFetcherConfig` exposes no `verifySsl` - setting, so an http-fetcher config cannot turn verification on. Do not use this fetcher - to retrieve anything whose authenticity matters over an untrusted network. - -* **`maxRedirects` does not apply to range requests.** The main `fetch` builds a - `RequestConfig` from `maxRedirects`; the `startRange`/`endRange` overload sets no - request config at all and therefore uses the client's defaults (redirects enabled). - A `maxRedirects: 0` setting does not stop redirects on a range fetch. - -* **The redirect host allowlist is currently inert.** `CustomRedirectStrategy` will - refuse a redirect to a host outside `allowedHostsForRedirect`, but that set is never - populated from any configuration path, and the check is skipped when the set is empty. - Do not rely on it to contain redirects. +* **TLS certificates are verified by default (`verifySsl: true`).** Earlier 4.0.0 + prereleases could not verify at all: the underlying client defaulted to `false` and + `HttpFetcherConfig` exposed no setting, so no configuration could turn verification on. ++ +To opt out -- self-signed internal certificates are the usual reason -- set `verifySsl` +to `false` on the fetcher: ++ +[source,json] +---- +{ + "fetchers": { + "my-http-fetcher": { + "plugin": "http-fetcher", + "config": { "verifySsl": false } + } + } +} +---- ++ +That installs an accept-everything trust strategy and `NoopHostnameVerifier`: any +certificate from any host is accepted, so anything on the network path can read and +alter what is fetched. Prefer adding your CA to the JVM truststore over turning this off. + +* **The redirect host allowlist is inert, and would not be trustworthy if it were not.** + `CustomRedirectStrategy` refuses a redirect to a host outside `allowedHostsForRedirect`, + but that set is never populated from any configuration path, and the check is skipped + when the set is empty. It also allows the redirect outright when the `Location` header + fails to parse as a URI, so the allowlist would have a bypass even once populated. Do + not rely on it to contain redirects. [#notes] == Notes diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java b/tika-pipes/tika-pipes-plugins/tika-pipes-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java index dad55c9e55..bbaa22aec7 100644 --- a/tika-pipes/tika-pipes-plugins/tika-pipes-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java @@ -162,15 +162,18 @@ public class HttpFetcher extends AbstractTikaExtension implements Fetcher, Range public TikaInputStream fetch(String fetchKey, Metadata metadata, ParseContext parseContext) throws IOException, TikaException { HttpFetcherConfig additionalHttpFetcherConfig = getAdditionalHttpFetcherConfig(parseContext); HttpGet get = new HttpGet(fetchKey); - RequestConfig requestConfig = RequestConfig + get.setConfig(buildRequestConfig()); + setHttpRequestHeaders(metadata, get); + putAdditionalHeadersOnRequest(additionalHttpFetcherConfig, get); + return execute(get, metadata, httpClient, true); + } + + private RequestConfig buildRequestConfig() { + return RequestConfig .custom() .setMaxRedirects(httpFetcherConfig.getMaxRedirects()) .setRedirectsEnabled(httpFetcherConfig.getMaxRedirects() > 0) .build(); - get.setConfig(requestConfig); - setHttpRequestHeaders(metadata, get); - putAdditionalHeadersOnRequest(additionalHttpFetcherConfig, get); - return execute(get, metadata, httpClient, true); } private void setHttpRequestHeaders(Metadata metadata, HttpGet get) { @@ -218,6 +221,11 @@ public class HttpFetcher extends AbstractTikaExtension implements Fetcher, Range ParseContext parseContext) throws IOException, TikaException { HttpFetcherConfig additionalHttpFetcherConfig = getAdditionalHttpFetcherConfig(parseContext); HttpGet get = new HttpGet(fetchKey); + // Same RequestConfig and headers as the whole-document fetch above. Without the + // config this used client defaults, which leave redirects ENABLED -- so + // maxRedirects: 0 did not stop redirects on a range fetch. + get.setConfig(buildRequestConfig()); + setHttpRequestHeaders(metadata, get); putAdditionalHeadersOnRequest(additionalHttpFetcherConfig, get); get.setHeader("Range", "bytes=" + startRange + "-" + endRange); @@ -466,7 +474,7 @@ public class HttpFetcher extends AbstractTikaExtension implements Fetcher, Range httpClientFactory.setRequestTimeoutMillis(httpFetcherConfig.getRequestTimeoutMillis()); } if (httpFetcherConfig.getConnectTimeoutMillis() != null) { - httpClientFactory.setSocketTimeoutMillis(httpFetcherConfig.getConnectTimeoutMillis()); + httpClientFactory.setConnectTimeoutMillis(httpFetcherConfig.getConnectTimeoutMillis()); } if (httpFetcherConfig.getMaxConnections() != null) { httpClientFactory.setMaxConnections(httpFetcherConfig.getMaxConnections()); @@ -474,6 +482,9 @@ public class HttpFetcher extends AbstractTikaExtension implements Fetcher, Range if (httpFetcherConfig.getMaxConnectionsPerRoute() != null) { httpClientFactory.setMaxConnectionsPerRoute(httpFetcherConfig.getMaxConnectionsPerRoute()); } + // Unreachable before: the factory had the setter, but nothing carried a config value + // to it, so TLS verification could not be turned on from any config. + httpClientFactory.setVerifySsl(httpFetcherConfig.isVerifySsl()); if (!StringUtils.isBlank(httpFetcherConfig.getAuthScheme())) { httpClientFactory.setUserName(httpFetcherConfig.getUserName()); httpClientFactory.setPassword(httpFetcherConfig.getPassword()); diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-http/src/main/java/org/apache/tika/pipes/fetcher/http/config/HttpFetcherConfig.java b/tika-pipes/tika-pipes-plugins/tika-pipes-http/src/main/java/org/apache/tika/pipes/fetcher/http/config/HttpFetcherConfig.java index dc4335f6fb..aba8a4c420 100644 --- a/tika-pipes/tika-pipes-plugins/tika-pipes-http/src/main/java/org/apache/tika/pipes/fetcher/http/config/HttpFetcherConfig.java +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-http/src/main/java/org/apache/tika/pipes/fetcher/http/config/HttpFetcherConfig.java @@ -51,6 +51,15 @@ public class HttpFetcherConfig { private Integer socketTimeoutMillis = 120000; private Long maxSpoolSize = -1L; private Integer maxRedirects = 0; + /** + * Verify server certificates and hostnames. Defaults to true, matching the Solr + * plugins; previously there was no field at all, so the factory's setting could not + * be reached from any config and verification could not be turned on. + * <p> + * Set to false to accept any certificate from any host -- needed for self-signed + * internal certs, and the only supported way to opt out. + */ + private boolean verifySsl = true; private List<String> httpHeaders = new ArrayList<>(); private HttpHeaders httpRequestHeaders = new HttpHeaders(); private Long overallTimeoutMillis = 120000L; @@ -269,4 +278,13 @@ public class HttpFetcherConfig { this.jwtPrivateKeyBase64 = jwtPrivateKeyBase64; return this; } + public boolean isVerifySsl() { + return verifySsl; + } + + public HttpFetcherConfig setVerifySsl(boolean verifySsl) { + this.verifySsl = verifySsl; + return this; + } + }
