This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new c728f17533 Upgrade httpclient to http.client5 and some random IT fixes
(#6902)
c728f17533 is described below
commit c728f175332a319d47e4676c6e94ca5ddb0ef54d
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Mon Mar 30 20:39:46 2026 +0200
Upgrade httpclient to http.client5 and some random IT fixes (#6902)
---
core/pom.xml | 10 --
.../apache/hop/core/util/HttpClientManager.java | 67 ++++++-----
.../org/apache/hop/core/util/HttpClientUtil.java | 60 +++++-----
.../hop/metadata/inject/HopMetadataInjector.java | 45 +++++++
.../apache/hop/server/ServerConnectionManager.java | 10 +-
.../metadata/inject/HopMetadataInjectorTest.java | 7 ++
.../java/org/apache/hop/core/HttpProtocol.java | 27 +++--
.../remote/RemoteExecutionInfoLocation.java | 2 +-
.../java/org/apache/hop/server/HopServerMeta.java | 131 ++++++++++-----------
.../java/org/apache/hop/www/BaseHttpServlet.java | 2 +-
.../java/org/apache/hop/server/HopServerTest.java | 82 ++++++-------
lib/pom.xml | 6 -
plugins/misc/rest/pom.xml | 10 +-
plugins/misc/rest/src/assembly/assembly.xml | 2 +-
.../dorisbulkloader/DorisStreamLoad.java | 44 +++----
.../execsqlrow/messages/messages_en_US.properties | 4 +-
.../transforms/execsqlrow/ExecSqlRowMetaTest.java | 19 +++
.../apache/hop/pipeline/transforms/http/Http.java | 91 +++++++-------
.../hop/pipeline/transforms/http/HttpData.java | 2 +-
.../hop/pipeline/transforms/http/HttpDataTest.java | 4 +-
.../transforms/http/HttpExecutionTest.java | 5 +-
.../hop/pipeline/transforms/http/HttpTest.java | 8 +-
plugins/transforms/httppost/pom.xml | 12 +-
.../hop/pipeline/transforms/httppost/HttpPost.java | 115 +++++++++---------
.../pipeline/transforms/httppost/HttpPostData.java | 2 +-
.../transforms/httppost/HttpPostExecutionTest.java | 7 +-
.../pipeline/transforms/httppost/HttpPostTest.java | 47 ++++----
.../pipeline/transforms/metainject/MetaInject.java | 32 +++--
.../transforms/metainject/MetaInjectDialog.java | 28 ++++-
plugins/transforms/rest/pom.xml | 4 +-
plugins/transforms/rest/src/assembly/assembly.xml | 2 +-
.../apache/hop/pipeline/transforms/rest/Rest.java | 4 +-
.../hop/pipeline/transforms/sort/SortRows.java | 24 +++-
.../hop/pipeline/transforms/sort/SortRowsTest.java | 112 ++++++++++++++++++
.../transforms/webservices/WebService.java | 85 +++++++------
.../pipeline/transforms/webservices/wsdl/Wsdl.java | 52 ++++----
.../transforms/webservices/WebServiceTest.java | 6 +-
.../transforms/xml/getxmldata/GetXmlData.java | 12 +-
.../hop/ui/www/service/WebServiceEditor.java | 2 +-
39 files changed, 701 insertions(+), 483 deletions(-)
diff --git a/core/pom.xml b/core/pom.xml
index 9dee1d413b..0877df7c52 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -126,16 +126,6 @@
</exclusion>
</exclusions>
</dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <exclusions>
- <exclusion>
- <groupId>*</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
diff --git a/core/src/main/java/org/apache/hop/core/util/HttpClientManager.java
b/core/src/main/java/org/apache/hop/core/util/HttpClientManager.java
index f6868202bf..e5fa258b0b 100644
--- a/core/src/main/java/org/apache/hop/core/util/HttpClientManager.java
+++ b/core/src/main/java/org/apache/hop/core/util/HttpClientManager.java
@@ -30,33 +30,34 @@ import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.TrustManagerFactory;
import javax.net.ssl.X509TrustManager;
+import org.apache.hc.client5.http.auth.AuthScope;
+import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
+import org.apache.hc.client5.http.config.RequestConfig;
+import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
+import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
+import org.apache.hc.client5.http.impl.classic.HttpClientBuilder;
+import org.apache.hc.client5.http.impl.classic.HttpClients;
+import org.apache.hc.client5.http.impl.io.BasicHttpClientConnectionManager;
+import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
+import org.apache.hc.client5.http.protocol.RedirectStrategy;
+import org.apache.hc.client5.http.socket.ConnectionSocketFactory;
+import org.apache.hc.client5.http.socket.PlainConnectionSocketFactory;
+import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
+import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactory;
+import org.apache.hc.core5.http.HttpHost;
+import org.apache.hc.core5.http.config.Registry;
+import org.apache.hc.core5.http.config.RegistryBuilder;
+import org.apache.hc.core5.ssl.SSLContexts;
+import org.apache.hc.core5.ssl.TrustStrategy;
+import org.apache.hc.core5.util.Timeout;
import org.apache.hop.core.logging.ILogChannel;
-import org.apache.http.HttpHost;
-import org.apache.http.auth.AuthScope;
-import org.apache.http.auth.UsernamePasswordCredentials;
-import org.apache.http.client.CredentialsProvider;
-import org.apache.http.client.RedirectStrategy;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.config.Registry;
-import org.apache.http.config.RegistryBuilder;
-import org.apache.http.conn.socket.ConnectionSocketFactory;
-import org.apache.http.conn.socket.PlainConnectionSocketFactory;
-import org.apache.http.conn.ssl.NoopHostnameVerifier;
-import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
-import org.apache.http.impl.client.BasicCredentialsProvider;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClientBuilder;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
-import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
-import org.apache.http.ssl.SSLContexts;
-import org.apache.http.ssl.TrustStrategy;
/**
- * Single entry point for all {@link org.apache.http.client.HttpClient
HttpClient instances} usages
- * in Hop projects. Contains {@link
org.apache.http.impl.conn.PoolingHttpClientConnectionManager
- * Connection pool} of 200 connections. Maximum connections per one route is
100. Provides inner
- * builder class for creating {@link org.apache.http.client.HttpClient
HttpClients}.
+ * Single entry point for all {@link
org.apache.hc.client5.http.classic.HttpClient HttpClient
+ * instances} usages in Hop projects. Contains {@link
+ * org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager
Connection pool} of 200
+ * connections. Maximum connections per one route is 100. Provides inner
builder class for creating
+ * {@link org.apache.hc.client5.http.classic.HttpClient HttpClients}.
*/
public class HttpClientManager {
private static final int CONNECTIONS_PER_ROUTE = 100;
@@ -88,7 +89,7 @@ public class HttpClientManager {
public class HttpClientBuilderFacade {
private RedirectStrategy redirectStrategy;
- private CredentialsProvider provider;
+ private BasicCredentialsProvider provider;
private int connectionTimeout;
private int socketTimeout;
private HttpHost proxy;
@@ -106,15 +107,17 @@ public class HttpClientManager {
public HttpClientBuilderFacade setCredentials(
String user, String password, AuthScope authScope) {
- CredentialsProvider provider = new BasicCredentialsProvider();
- UsernamePasswordCredentials credentials = new
UsernamePasswordCredentials(user, password);
+ BasicCredentialsProvider provider = new BasicCredentialsProvider();
+ char[] passwordChars = password != null ? password.toCharArray() : new
char[0];
+ UsernamePasswordCredentials credentials =
+ new UsernamePasswordCredentials(user, passwordChars);
provider.setCredentials(authScope, credentials);
this.provider = provider;
return this;
}
public HttpClientBuilderFacade setCredentials(String user, String
password) {
- return setCredentials(user, password, AuthScope.ANY);
+ return setCredentials(user, password, new AuthScope(null, null, -1,
null, null));
}
public HttpClientBuilderFacade setProxy(String proxyHost, int proxyPort) {
@@ -123,7 +126,7 @@ public class HttpClientManager {
}
public HttpClientBuilderFacade setProxy(String proxyHost, int proxyPort,
String scheme) {
- this.proxy = new HttpHost(proxyHost, proxyPort, scheme);
+ this.proxy = new HttpHost(scheme, proxyHost, proxyPort);
return this;
}
@@ -157,7 +160,7 @@ public class HttpClientManager {
BasicHttpClientConnectionManager connectionManager =
new BasicHttpClientConnectionManager(socketFactoryRegistry);
-
httpClientBuilder.setSSLSocketFactory(sslsf).setConnectionManager(connectionManager);
+ httpClientBuilder.setConnectionManager(connectionManager);
}
public CloseableHttpClient build() {
@@ -166,10 +169,10 @@ public class HttpClientManager {
RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
if (socketTimeout > 0) {
- requestConfigBuilder.setSocketTimeout(socketTimeout);
+
requestConfigBuilder.setResponseTimeout(Timeout.ofMilliseconds(socketTimeout));
}
if (connectionTimeout > 0) {
- requestConfigBuilder.setConnectTimeout(socketTimeout);
+
requestConfigBuilder.setConnectTimeout(Timeout.ofMilliseconds(connectionTimeout));
}
if (proxy != null) {
requestConfigBuilder.setProxy(proxy);
diff --git a/core/src/main/java/org/apache/hop/core/util/HttpClientUtil.java
b/core/src/main/java/org/apache/hop/core/util/HttpClientUtil.java
index 2da12fc20f..15e186b55b 100644
--- a/core/src/main/java/org/apache/hop/core/util/HttpClientUtil.java
+++ b/core/src/main/java/org/apache/hop/core/util/HttpClientUtil.java
@@ -22,22 +22,20 @@ import java.io.InputStream;
import java.net.URLDecoder;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpHost;
-import org.apache.http.HttpResponse;
-import org.apache.http.auth.AuthScope;
-import org.apache.http.auth.UsernamePasswordCredentials;
-import org.apache.http.client.AuthCache;
-import org.apache.http.client.CredentialsProvider;
-import org.apache.http.client.protocol.HttpClientContext;
-import org.apache.http.impl.auth.BasicScheme;
-import org.apache.http.impl.client.BasicAuthCache;
-import org.apache.http.impl.client.BasicCredentialsProvider;
-import org.apache.http.util.EntityUtils;
+import org.apache.hc.client5.http.auth.AuthCache;
+import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
+import org.apache.hc.client5.http.impl.auth.BasicAuthCache;
+import org.apache.hc.client5.http.impl.auth.BasicScheme;
+import org.apache.hc.client5.http.protocol.HttpClientContext;
+import org.apache.hc.core5.http.ClassicHttpResponse;
+import org.apache.hc.core5.http.HttpEntity;
+import org.apache.hc.core5.http.HttpHost;
+import org.apache.hc.core5.http.ParseException;
+import org.apache.hc.core5.http.io.entity.EntityUtils;
/**
* Utility class contained useful methods while working with {@link
- * org.apache.http.client.HttpClient HttpClient}
+ * org.apache.hc.client5.http.classic.HttpClient HttpClient}
*/
public class HttpClientUtil {
@@ -48,7 +46,7 @@ public class HttpClientUtil {
* @return HttpEntity in String representation using "UTF-8" encoding
* @throws IOException
*/
- public static String responseToString(HttpResponse response) throws
IOException {
+ public static String responseToString(ClassicHttpResponse response) throws
IOException {
return responseToString(response,
Charset.forName(StandardCharsets.UTF_8.name()));
}
@@ -58,7 +56,8 @@ public class HttpClientUtil {
* @return HttpEntity in decoded String representation using provided charset
* @throws IOException
*/
- public static String responseToString(HttpResponse response, Charset
charset) throws IOException {
+ public static String responseToString(ClassicHttpResponse response, Charset
charset)
+ throws IOException {
return responseToString(response, charset, false);
}
@@ -69,10 +68,15 @@ public class HttpClientUtil {
* @return HttpEntity in String representation using provided charset
* @throws IOException
*/
- public static String responseToString(HttpResponse response, Charset
charset, boolean decode)
- throws IOException {
+ public static String responseToString(
+ ClassicHttpResponse response, Charset charset, boolean decode) throws
IOException {
HttpEntity entity = response.getEntity();
- String result = EntityUtils.toString(entity, charset);
+ String result;
+ try {
+ result = EntityUtils.toString(entity, charset);
+ } catch (ParseException e) {
+ throw new IOException("Unable to parse HTTP response entity", e);
+ }
EntityUtils.consume(entity);
if (decode) {
result = URLDecoder.decode(result, StandardCharsets.UTF_8.name());
@@ -80,11 +84,11 @@ public class HttpClientUtil {
return result;
}
- public static InputStream responseToInputStream(HttpResponse response)
throws IOException {
+ public static InputStream responseToInputStream(ClassicHttpResponse
response) throws IOException {
return response.getEntity().getContent();
}
- public static byte[] responseToByteArray(HttpResponse response) throws
IOException {
+ public static byte[] responseToByteArray(ClassicHttpResponse response)
throws IOException {
return EntityUtils.toByteArray(response.getEntity());
}
@@ -96,23 +100,17 @@ public class HttpClientUtil {
* @param user
* @param password
* @param schema
- * @return {@link org.apache.http.client.protocol.HttpClientContext
HttpClientContext}
+ * @return {@link org.apache.hc.client5.http.protocol.HttpClientContext
HttpClientContext}
*/
public static HttpClientContext createPreemptiveBasicAuthentication(
String host, int port, String user, String password, String schema) {
HttpClientContext localContext = null;
try {
- HttpHost target = new HttpHost(host, port, schema);
- CredentialsProvider credsProvider = new BasicCredentialsProvider();
- credsProvider.setCredentials(
- new AuthScope(target.getHostName(), target.getPort()),
- new UsernamePasswordCredentials(user, password));
-
- // Create AuthCache instance
+ HttpHost target = new HttpHost(schema, host, port);
+ char[] passwordChars = password != null ? password.toCharArray() : new
char[0];
AuthCache authCache = new BasicAuthCache();
- // Generate BASIC scheme object and add it to the local
- // auth cache
BasicScheme basicAuth = new BasicScheme();
+ basicAuth.initPreemptive(new UsernamePasswordCredentials(user,
passwordChars));
authCache.put(target, basicAuth);
// Add AuthCache to the execution context
@@ -131,7 +129,7 @@ public class HttpClientUtil {
* @param port
* @param user
* @param password
- * @return {@link org.apache.http.client.protocol.HttpClientContext
HttpClientContext}
+ * @return {@link org.apache.hc.client5.http.protocol.HttpClientContext
HttpClientContext}
*/
public static HttpClientContext createPreemptiveBasicAuthentication(
String host, int port, String user, String password) {
diff --git
a/core/src/main/java/org/apache/hop/metadata/inject/HopMetadataInjector.java
b/core/src/main/java/org/apache/hop/metadata/inject/HopMetadataInjector.java
index c60b422820..093fc42372 100644
--- a/core/src/main/java/org/apache/hop/metadata/inject/HopMetadataInjector.java
+++ b/core/src/main/java/org/apache/hop/metadata/inject/HopMetadataInjector.java
@@ -66,6 +66,51 @@ public class HopMetadataInjector {
return map;
}
+ /**
+ * Returns true if {@code targetKey} maps to a {@link HopMetadataProperty}
that is injected via
+ * the simple key map (not via a list / {@link
HopMetadataProperty#injectionGroupKey()}). Such
+ * keys are not present in {@link #findInjectionGroupKeys(Class)}.
+ *
+ * @param objectClass class annotated with HopMetadataProperty (e.g. a
transform meta class)
+ * @param targetKey injection key to test (e.g. from MetaInject mapping)
+ */
+ public static boolean isTopLevelInjectionKey(Class<?> objectClass, String
targetKey)
+ throws HopException {
+ if (objectClass == null || StringUtils.isEmpty(targetKey)) {
+ return false;
+ }
+ List<Field> fields =
+ ReflectionUtil.findAllFields(objectClass, new
MetadataPropertyKeyFunction());
+ for (Field field : fields) {
+ HopMetadataProperty property =
field.getAnnotation(HopMetadataProperty.class);
+ if (property == null || property.isExcludedFromInjection()) {
+ continue;
+ }
+ Class<?> fieldClass = field.getType();
+ String injectionGroup = Const.NVL(property.injectionGroupKey(),
property.groupKey());
+ if (StringUtils.isNotEmpty(injectionGroup)) {
+ continue;
+ }
+ if (fieldClass.equals(List.class)) {
+ continue;
+ }
+ String injectionKey =
+ Const.coalesce(property.injectionKey(), property.key(),
field.getName());
+ if (targetKey.equals(injectionKey)) {
+ return true;
+ }
+ if (!fieldClass.isEnum()
+ && !fieldClass.isPrimitive()
+ && !fieldClass.equals(String.class)
+ && !fieldClass.isArray()) {
+ if (isTopLevelInjectionKey(fieldClass, targetKey)) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
/**
* For the given object we retrieve the mappings between injection keys and
the group to which
* they belong.
diff --git
a/core/src/main/java/org/apache/hop/server/ServerConnectionManager.java
b/core/src/main/java/org/apache/hop/server/ServerConnectionManager.java
index 40ebdec1d0..ea71312508 100644
--- a/core/src/main/java/org/apache/hop/server/ServerConnectionManager.java
+++ b/core/src/main/java/org/apache/hop/server/ServerConnectionManager.java
@@ -23,9 +23,9 @@ import java.security.cert.X509Certificate;
import javax.net.ssl.KeyManager;
import javax.net.ssl.SSLContext;
import javax.net.ssl.X509TrustManager;
-import org.apache.http.client.HttpClient;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
+import org.apache.hc.client5.http.impl.classic.HttpClients;
+import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
/**
* Encapsulates the Apache commons HTTP connection manager with a singleton.
We can use this to
@@ -69,12 +69,12 @@ public class ServerConnectionManager {
return serverConnectionManager;
}
- public HttpClient createHttpClient() {
+ public CloseableHttpClient createHttpClient() {
return HttpClients.custom().setConnectionManager(manager).build();
}
public void shutdown() {
- manager.shutdown();
+ manager.close();
}
private static X509TrustManager getDefaultTrustManager() {
diff --git
a/core/src/test/java/org/apache/hop/metadata/inject/HopMetadataInjectorTest.java
b/core/src/test/java/org/apache/hop/metadata/inject/HopMetadataInjectorTest.java
index b2edc2b768..8d95945e55 100644
---
a/core/src/test/java/org/apache/hop/metadata/inject/HopMetadataInjectorTest.java
+++
b/core/src/test/java/org/apache/hop/metadata/inject/HopMetadataInjectorTest.java
@@ -19,6 +19,7 @@
package org.apache.hop.metadata.inject;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -109,6 +110,12 @@ class HopMetadataInjectorTest {
assertTrue(keys.contains("LAST_NAME"));
}
+ @Test
+ void testTopLevelInjectionKey() throws Exception {
+ assertTrue(HopMetadataInjector.isTopLevelInjectionKey(Company.class,
"NAME"));
+ assertFalse(HopMetadataInjector.isTopLevelInjectionKey(Company.class,
"FIRST_NAME"));
+ }
+
@Test
void testSampleMetaInjection() throws Exception {
Map<String, Object> injectionKeyMap = new HashMap<>();
diff --git a/engine/src/main/java/org/apache/hop/core/HttpProtocol.java
b/engine/src/main/java/org/apache/hop/core/HttpProtocol.java
index 65902c41fb..62b17e7a29 100644
--- a/engine/src/main/java/org/apache/hop/core/HttpProtocol.java
+++ b/engine/src/main/java/org/apache/hop/core/HttpProtocol.java
@@ -19,13 +19,17 @@ package org.apache.hop.core;
import java.io.IOException;
import java.io.InputStreamReader;
+import java.net.URI;
+import org.apache.hc.client5.http.auth.AuthenticationException;
+import org.apache.hc.client5.http.classic.HttpClient;
+import org.apache.hc.client5.http.classic.methods.HttpGet;
+import org.apache.hc.client5.http.protocol.HttpClientContext;
+import org.apache.hc.core5.http.ClassicHttpResponse;
+import org.apache.hc.core5.http.HttpHost;
+import org.apache.hc.core5.http.HttpStatus;
import org.apache.hop.core.util.HttpClientManager;
+import org.apache.hop.core.util.HttpClientUtil;
import org.apache.hop.core.util.Utils;
-import org.apache.http.HttpResponse;
-import org.apache.http.HttpStatus;
-import org.apache.http.auth.AuthenticationException;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.HttpGet;
/**
* HTTP
@@ -70,16 +74,25 @@ public class HttpProtocol {
HttpClient httpClient;
HttpGet getMethod = new HttpGet(urlAsString);
+ HttpClientContext clientContext = HttpClientContext.create();
if (!Utils.isEmpty(username)) {
HttpClientManager.HttpClientBuilderFacade clientBuilder =
HttpClientManager.getInstance().createBuilder();
clientBuilder.setCredentials(username, password);
httpClient = clientBuilder.build();
+ HttpHost origin = HttpHost.create(URI.create(urlAsString));
+ HttpClientContext preemptive =
+ HttpClientUtil.createPreemptiveBasicAuthentication(
+ origin.getHostName(), origin.getPort(), username, password,
origin.getSchemeName());
+ if (preemptive != null) {
+ clientContext = preemptive;
+ }
} else {
httpClient = HttpClientManager.getInstance().createDefaultClient();
}
- HttpResponse httpResponse = httpClient.execute(getMethod);
- int statusCode = httpResponse.getStatusLine().getStatusCode();
+ ClassicHttpResponse httpResponse =
+ (ClassicHttpResponse) httpClient.execute(getMethod, clientContext);
+ int statusCode = httpResponse.getCode();
StringBuilder bodyBuffer = new StringBuilder();
if (statusCode != -1) {
diff --git
a/engine/src/main/java/org/apache/hop/execution/remote/RemoteExecutionInfoLocation.java
b/engine/src/main/java/org/apache/hop/execution/remote/RemoteExecutionInfoLocation.java
index 2126abdaff..a581c9c02c 100644
---
a/engine/src/main/java/org/apache/hop/execution/remote/RemoteExecutionInfoLocation.java
+++
b/engine/src/main/java/org/apache/hop/execution/remote/RemoteExecutionInfoLocation.java
@@ -26,6 +26,7 @@ import java.util.List;
import lombok.Getter;
import lombok.Setter;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hc.core5.net.URIBuilder;
import org.apache.hop.core.Const;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.gui.plugin.GuiElementType;
@@ -49,7 +50,6 @@ import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.server.HopServerMeta;
import org.apache.hop.www.GetExecutionInfoServlet;
import org.apache.hop.www.RegisterExecutionInfoServlet;
-import org.apache.http.client.utils.URIBuilder;
@GuiPlugin(description = "File execution information location GUI elements")
@ExecutionInfoLocationPlugin(
diff --git a/engine/src/main/java/org/apache/hop/server/HopServerMeta.java
b/engine/src/main/java/org/apache/hop/server/HopServerMeta.java
index 7babc35b5a..f7153d545c 100644
--- a/engine/src/main/java/org/apache/hop/server/HopServerMeta.java
+++ b/engine/src/main/java/org/apache/hop/server/HopServerMeta.java
@@ -36,6 +36,30 @@ import javax.net.ssl.SSLContext;
import lombok.Getter;
import lombok.Setter;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hc.client5.http.auth.AuthCache;
+import org.apache.hc.client5.http.auth.AuthScope;
+import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
+import org.apache.hc.client5.http.classic.HttpClient;
+import org.apache.hc.client5.http.config.RequestConfig;
+import org.apache.hc.client5.http.impl.auth.BasicAuthCache;
+import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
+import org.apache.hc.client5.http.impl.auth.BasicScheme;
+import org.apache.hc.client5.http.impl.classic.HttpClients;
+import org.apache.hc.client5.http.protocol.HttpClientContext;
+import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
+import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactory;
+import org.apache.hc.client5.http.ssl.TrustSelfSignedStrategy;
+import org.apache.hc.core5.http.ClassicHttpRequest;
+import org.apache.hc.core5.http.ClassicHttpResponse;
+import org.apache.hc.core5.http.ContentType;
+import org.apache.hc.core5.http.HttpEntity;
+import org.apache.hc.core5.http.HttpHost;
+import org.apache.hc.core5.http.HttpStatus;
+import org.apache.hc.core5.http.io.entity.ByteArrayEntity;
+import org.apache.hc.core5.http.io.entity.InputStreamEntity;
+import org.apache.hc.core5.http.message.BasicHeader;
+import org.apache.hc.core5.ssl.SSLContexts;
+import org.apache.hc.core5.ssl.TrustStrategy;
import org.apache.hop.core.Const;
import org.apache.hop.core.Result;
import org.apache.hop.core.encryption.Encr;
@@ -71,34 +95,6 @@ import org.apache.hop.www.StartWorkflowServlet;
import org.apache.hop.www.StopPipelineServlet;
import org.apache.hop.www.StopWorkflowServlet;
import org.apache.hop.www.WebResult;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpHost;
-import org.apache.http.HttpResponse;
-import org.apache.http.HttpStatus;
-import org.apache.http.StatusLine;
-import org.apache.http.auth.AuthScope;
-import org.apache.http.auth.UsernamePasswordCredentials;
-import org.apache.http.client.AuthCache;
-import org.apache.http.client.ClientProtocolException;
-import org.apache.http.client.CredentialsProvider;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.methods.HttpUriRequest;
-import org.apache.http.client.protocol.HttpClientContext;
-import org.apache.http.conn.ssl.NoopHostnameVerifier;
-import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
-import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
-import org.apache.http.conn.ssl.TrustStrategy;
-import org.apache.http.entity.ByteArrayEntity;
-import org.apache.http.entity.InputStreamEntity;
-import org.apache.http.impl.auth.BasicScheme;
-import org.apache.http.impl.client.BasicAuthCache;
-import org.apache.http.impl.client.BasicCredentialsProvider;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.http.message.BasicHeader;
-import org.apache.http.ssl.SSLContexts;
import org.w3c.dom.Node;
@Getter
@@ -387,13 +383,14 @@ public class HopServerMeta extends HopMetadataBase
implements Cloneable, IXml, I
return result;
}
- HttpPost buildSendXmlMethod(IVariables variables, byte[] content, String
service) {
+ org.apache.hc.client5.http.classic.methods.HttpPost buildSendXmlMethod(
+ IVariables variables, byte[] content, String service) {
String encoding = Const.XML_ENCODING;
return buildSendMethod(variables, content, encoding, service, "text/xml");
}
// Method is defined as package-protected in order to be accessible by unit
tests
- HttpPost buildSendMethod(
+ org.apache.hc.client5.http.classic.methods.HttpPost buildSendMethod(
IVariables variables, byte[] content, String encoding, String service,
String contentType) {
// Prepare HTTP put
//
@@ -401,11 +398,12 @@ public class HopServerMeta extends HopMetadataBase
implements Cloneable, IXml, I
if (log.isDebug()) {
log.logDebug(BaseMessages.getString(PKG, "HopServer.DEBUG_ConnectingTo",
urlString));
}
- HttpPost postMethod = new HttpPost(urlString);
+ org.apache.hc.client5.http.classic.methods.HttpPost postMethod =
+ new org.apache.hc.client5.http.classic.methods.HttpPost(urlString);
// Request content will be retrieved directly from the input stream
//
- HttpEntity entity = new ByteArrayEntity(content);
+ HttpEntity entity = new ByteArrayEntity(content,
ContentType.APPLICATION_OCTET_STREAM);
postMethod.setEntity(entity);
postMethod.addHeader(new BasicHeader("Accept", contentType + ";charset=" +
encoding));
@@ -415,13 +413,11 @@ public class HopServerMeta extends HopMetadataBase
implements Cloneable, IXml, I
public String sendXml(IVariables variables, String xml, String service)
throws Exception {
String encoding = getXmlEncoding(xml);
- HttpPost method =
+ org.apache.hc.client5.http.classic.methods.HttpPost method =
buildSendMethod(variables, xml.getBytes(encoding), encoding, service,
"text/xml");
try {
return executeAuth(variables, method);
} finally {
- // Release current connection to the connection pool once you are done
- method.releaseConnection();
if (log.isDetailed()) {
log.logDetailed(
BaseMessages.getString(
@@ -432,13 +428,11 @@ public class HopServerMeta extends HopMetadataBase
implements Cloneable, IXml, I
public String sendJson(IVariables variables, String json, String service)
throws Exception {
String encoding = Const.XML_ENCODING;
- HttpPost method =
+ org.apache.hc.client5.http.classic.methods.HttpPost method =
buildSendMethod(variables, json.getBytes(encoding), encoding, service,
"application/json");
try {
return executeAuth(variables, method);
} finally {
- // Release current connection to the connection pool once you are done
- method.releaseConnection();
if (log.isDetailed()) {
log.logDetailed(
BaseMessages.getString(
@@ -459,7 +453,7 @@ public class HopServerMeta extends HopMetadataBase
implements Cloneable, IXml, I
/** Throws if not ok */
private void handleStatus(
- IVariables variables, HttpUriRequest method, StatusLine statusLine, int
status)
+ IVariables variables, ClassicHttpRequest method, ClassicHttpResponse
response, int status)
throws HopException {
if (status >= 300) {
String message;
@@ -475,14 +469,15 @@ public class HopServerMeta extends HopMetadataBase
implements Cloneable, IXml, I
message =
String.format(
"HTTP Status %d - %s - %s",
- status, method.getURI().toString(),
statusLine.getReasonPhrase());
+ status, method.getRequestUri(), response.getReasonPhrase());
}
throw new HopException(message);
}
}
// Method is defined as package-protected in order to be accessible by unit
tests
- HttpPost buildSendExportMethod(IVariables variables, String type, String
load, InputStream is) {
+ org.apache.hc.client5.http.classic.methods.HttpPost buildSendExportMethod(
+ IVariables variables, String type, String load, InputStream is) {
String serviceUrl = RegisterPackageServlet.CONTEXT_PATH;
if (type != null && load != null) {
serviceUrl +=
@@ -501,8 +496,9 @@ public class HopServerMeta extends HopMetadataBase
implements Cloneable, IXml, I
log.logDebug(BaseMessages.getString(PKG, "HopServer.DEBUG_ConnectingTo",
urlString));
}
- HttpPost method = new HttpPost(urlString);
- method.setEntity(new InputStreamEntity(is));
+ org.apache.hc.client5.http.classic.methods.HttpPost method =
+ new org.apache.hc.client5.http.classic.methods.HttpPost(urlString);
+ method.setEntity(new InputStreamEntity(is,
ContentType.APPLICATION_OCTET_STREAM));
method.addHeader(new BasicHeader("Content-Type", "binary/zip"));
return method;
@@ -522,12 +518,11 @@ public class HopServerMeta extends HopMetadataBase
implements Cloneable, IXml, I
// Request content will be retrieved directly from the input stream
try (InputStream is =
HopVfs.getInputStream(HopVfs.getFileObject(filename))) {
// Execute request
- HttpPost method = buildSendExportMethod(variables, type, load, is);
+ org.apache.hc.client5.http.classic.methods.HttpPost method =
+ buildSendExportMethod(variables, type, load, is);
try {
return executeAuth(variables, method);
} finally {
- // Release current connection to the connection pool once you are done
- method.releaseConnection();
if (log.isDetailed()) {
log.logDetailed(
BaseMessages.getString(
@@ -549,16 +544,17 @@ public class HopServerMeta extends HopMetadataBase
implements Cloneable, IXml, I
* @throws ClientProtocolException
* @throws HopException if response not ok
*/
- private String executeAuth(IVariables variables, HttpUriRequest method)
+ private String executeAuth(IVariables variables, ClassicHttpRequest method)
throws IOException, HopException {
- HttpResponse httpResponse = getHttpClient().execute(method,
getAuthContext(variables));
+ ClassicHttpResponse httpResponse =
+ (ClassicHttpResponse) getHttpClient().execute(method,
getAuthContext(variables));
return getResponse(variables, method, httpResponse);
}
- private String getResponse(IVariables variables, HttpUriRequest method,
HttpResponse httpResponse)
+ private String getResponse(
+ IVariables variables, ClassicHttpRequest method, ClassicHttpResponse
httpResponse)
throws IOException, HopException {
- StatusLine statusLine = httpResponse.getStatusLine();
- int statusCode = statusLine.getStatusCode();
+ int statusCode = httpResponse.getCode();
// The status code
if (log.isDebug()) {
log.logDebug(
@@ -572,7 +568,7 @@ public class HopServerMeta extends HopMetadataBase
implements Cloneable, IXml, I
}
// throw if not ok
- handleStatus(variables, method, statusLine, statusCode);
+ handleStatus(variables, method, httpResponse, statusCode);
return responseBody;
}
@@ -585,17 +581,20 @@ public class HopServerMeta extends HopMetadataBase
implements Cloneable, IXml, I
String password =
Encr.decryptPasswordOptionallyEncrypted(variables.resolve(this.password));
String proxyHost = variables.resolve(proxyHostname);
- CredentialsProvider provider = new BasicCredentialsProvider();
- UsernamePasswordCredentials credentials = new
UsernamePasswordCredentials(userName, password);
+ BasicCredentialsProvider provider = new BasicCredentialsProvider();
+ char[] passwordChars = password != null ? password.toCharArray() : new
char[0];
+ UsernamePasswordCredentials credentials =
+ new UsernamePasswordCredentials(userName, passwordChars);
if (!Utils.isEmpty(proxyHost) && host.equals("localhost")) {
host = "127.0.0.1";
}
provider.setCredentials(new AuthScope(host, port), credentials);
context.setCredentialsProvider(provider);
// Generate BASIC scheme object and add it to the local auth cache
- HttpHost target = new HttpHost(host, port, isSslMode() ? HTTPS : HTTP);
+ HttpHost target = new HttpHost(isSslMode() ? HTTPS : HTTP, host, port);
AuthCache authCache = new BasicAuthCache();
BasicScheme basicAuth = new BasicScheme();
+ basicAuth.initPreemptive(credentials);
authCache.put(target, basicAuth);
context.setAuthCache(authCache);
}
@@ -613,7 +612,7 @@ public class HopServerMeta extends HopMetadataBase
implements Cloneable, IXml, I
if (!Utils.isEmpty(nonProxyHosts) && hostName.matches(nonProxyHosts)) {
return;
}
- HttpHost httpHost = new HttpHost(proxyHost, Integer.parseInt(proxyPort));
+ HttpHost httpHost = new HttpHost(HTTP, proxyHost,
Integer.parseInt(proxyPort));
RequestConfig requestConfig =
RequestConfig.custom().setProxy(httpHost).build();
@@ -687,9 +686,10 @@ public class HopServerMeta extends HopMetadataBase
implements Cloneable, IXml, I
}
// Method is defined as package-protected in order to be accessible by unit
tests
- HttpGet buildExecuteServiceMethod(
+ org.apache.hc.client5.http.classic.methods.HttpGet buildExecuteServiceMethod(
IVariables variables, String service, Map<String, String> headerValues) {
- HttpGet method = new HttpGet(constructUrl(variables, service));
+ org.apache.hc.client5.http.classic.methods.HttpGet method =
+ new
org.apache.hc.client5.http.classic.methods.HttpGet(constructUrl(variables,
service));
for (String key : headerValues.keySet()) {
method.setHeader(key, headerValues.get(key));
@@ -700,12 +700,13 @@ public class HopServerMeta extends HopMetadataBase
implements Cloneable, IXml, I
public String execService(IVariables variables, String service, Map<String,
String> headerValues)
throws Exception {
// Prepare HTTP get
- HttpGet method = buildExecuteServiceMethod(variables, service,
headerValues);
+ org.apache.hc.client5.http.classic.methods.HttpGet method =
+ buildExecuteServiceMethod(variables, service, headerValues);
// Execute request
try {
- HttpResponse httpResponse = getHttpClient().execute(method,
getAuthContext(variables));
- StatusLine statusLine = httpResponse.getStatusLine();
- int statusCode = statusLine.getStatusCode();
+ ClassicHttpResponse httpResponse =
+ (ClassicHttpResponse) getHttpClient().execute(method,
getAuthContext(variables));
+ int statusCode = httpResponse.getCode();
// The status code
if (log.isDebug()) {
@@ -731,13 +732,11 @@ public class HopServerMeta extends HopMetadataBase
implements Cloneable, IXml, I
throw new HopException(
String.format(
"HTTP Status %d - %s - %s",
- statusCode, method.getURI().toString(),
statusLine.getReasonPhrase()));
+ statusCode, method.getRequestUri(),
httpResponse.getReasonPhrase()));
}
return responseBody;
} finally {
- // Release current connection to the connection pool once you are done
- method.releaseConnection();
if (log.isDetailed()) {
log.logDetailed(
BaseMessages.getString(PKG, "HopServer.DETAILED_ExecutedService",
service, hostname));
@@ -757,7 +756,7 @@ public class HopServerMeta extends HopMetadataBase
implements Cloneable, IXml, I
SSLConnectionSocketFactory socketFactory =
new SSLConnectionSocketFactory(sslContext,
NoopHostnameVerifier.INSTANCE);
- return HttpClients.custom().setSSLSocketFactory(socketFactory).build();
+ return HttpClients.custom().setConnectionManagerShared(true).build();
} else {
// Connect using a regular HTTP connection
//
diff --git a/engine/src/main/java/org/apache/hop/www/BaseHttpServlet.java
b/engine/src/main/java/org/apache/hop/www/BaseHttpServlet.java
index 42c1c05f2b..b61cf1a2e1 100644
--- a/engine/src/main/java/org/apache/hop/www/BaseHttpServlet.java
+++ b/engine/src/main/java/org/apache/hop/www/BaseHttpServlet.java
@@ -29,13 +29,13 @@ import java.io.Serial;
import java.nio.charset.StandardCharsets;
import lombok.Getter;
import lombok.Setter;
+import org.apache.hc.core5.http.ContentType;
import org.apache.hop.core.Const;
import org.apache.hop.core.logging.ILogChannel;
import org.apache.hop.core.logging.LogChannel;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.core.variables.Variables;
import org.apache.hop.core.xml.XmlHandler;
-import org.apache.http.entity.ContentType;
public class BaseHttpServlet extends HttpServlet {
@Serial protected static final long serialVersionUID = -1348342810327662788L;
diff --git a/engine/src/test/java/org/apache/hop/server/HopServerTest.java
b/engine/src/test/java/org/apache/hop/server/HopServerTest.java
index 0d5445cefa..dd6f764448 100644
--- a/engine/src/test/java/org/apache/hop/server/HopServerTest.java
+++ b/engine/src/test/java/org/apache/hop/server/HopServerTest.java
@@ -39,6 +39,19 @@ import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
+import org.apache.hc.client5.http.auth.AuthCache;
+import org.apache.hc.client5.http.auth.AuthScope;
+import org.apache.hc.client5.http.classic.HttpClient;
+import org.apache.hc.client5.http.classic.methods.HttpGet;
+import org.apache.hc.client5.http.classic.methods.HttpPost;
+import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse;
+import org.apache.hc.client5.http.protocol.HttpClientContext;
+import org.apache.hc.core5.http.ClassicHttpRequest;
+import org.apache.hc.core5.http.ClassicHttpResponse;
+import org.apache.hc.core5.http.HttpEntity;
+import org.apache.hc.core5.http.HttpHost;
+import org.apache.hc.core5.http.HttpStatus;
+import org.apache.hc.core5.http.protocol.HttpContext;
import org.apache.hop.core.Const;
import org.apache.hop.core.encryption.Encr;
import org.apache.hop.core.encryption.TwoWayPasswordEncoderPluginType;
@@ -48,21 +61,6 @@ import org.apache.hop.core.util.EnvUtil;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.core.variables.Variables;
import org.apache.hop.utils.TestUtils;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpHost;
-import org.apache.http.HttpResponse;
-import org.apache.http.HttpStatus;
-import org.apache.http.StatusLine;
-import org.apache.http.auth.AuthScope;
-import org.apache.http.auth.Credentials;
-import org.apache.http.client.AuthCache;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.methods.HttpUriRequest;
-import org.apache.http.client.protocol.HttpClientContext;
-import org.apache.http.protocol.HttpContext;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
@@ -100,10 +98,7 @@ class HopServerTest {
// mock response
CloseableHttpResponse closeableHttpResponseMock =
mock(CloseableHttpResponse.class);
- // mock status line
- StatusLine statusLineMock = mock(StatusLine.class);
- doReturn(HttpStatus.SC_NOT_FOUND).when(statusLineMock).getStatusCode();
- doReturn(statusLineMock).when(closeableHttpResponseMock).getStatusLine();
+
doReturn(HttpStatus.SC_NOT_FOUND).when(closeableHttpResponseMock).getCode();
// mock entity
HttpEntity httpEntityMock = mock(HttpEntity.class);
@@ -121,11 +116,9 @@ class HopServerTest {
doReturn("response_body").when(hopServer).getResponseBodyAsString(nullable(InputStream.class));
}
- private HttpResponse mockResponse(int statusCode, String entityText) throws
IOException {
- HttpResponse resp = mock(HttpResponse.class);
- StatusLine status = mock(StatusLine.class);
- when(status.getStatusCode()).thenReturn(statusCode);
- when(resp.getStatusLine()).thenReturn(status);
+ private ClassicHttpResponse mockResponse(int statusCode, String entityText)
throws IOException {
+ ClassicHttpResponse resp = mock(ClassicHttpResponse.class);
+ when(resp.getCode()).thenReturn(statusCode);
HttpEntity entity = mock(HttpEntity.class);
when(entity.getContent())
.thenReturn(new
ByteArrayInputStream(entityText.getBytes(StandardCharsets.UTF_8)));
@@ -139,14 +132,14 @@ class HopServerTest {
HttpGet httpGetMock = mock(HttpGet.class);
URI uriMock = new URI(nonExistingAppName);
- doReturn(uriMock).when(httpGetMock).getURI();
+ doReturn(uriMock).when(httpGetMock).getUri();
HttpClient clientMock = mock(HttpClient.class);
- when(clientMock.execute(any(HttpUriRequest.class), any(HttpContext.class)))
+ when(clientMock.execute(any(ClassicHttpRequest.class),
any(HttpContext.class)))
.then(
invocation -> {
- HttpUriRequest request = invocation.getArgument(0);
- if (request.getURI().equals(uriMock)) {
+ ClassicHttpRequest request = invocation.getArgument(0);
+ if (request.getUri().equals(uriMock)) {
return mockResponse(404, "");
}
return mockResponse(200, "");
@@ -170,7 +163,7 @@ class HopServerTest {
hopServer.setUsername("userNAmeStub");
HttpPost httpPostMock = mock(HttpPost.class);
URI uriMock = new URI("fake");
- doReturn(uriMock).when(httpPostMock).getURI();
+ doReturn(uriMock).when(httpPostMock).getUri();
doReturn(httpPostMock)
.when(hopServer)
.buildSendXmlMethod(any(Variables.class), any(byte[].class),
anyString());
@@ -184,7 +177,7 @@ class HopServerTest {
hopServer.setUsername("userNAmeStub");
HttpPost httpPostMock = mock(HttpPost.class);
URI uriMock = new URI("fake");
- doReturn(uriMock).when(httpPostMock).getURI();
+ doReturn(uriMock).when(httpPostMock).getUri();
doReturn(httpPostMock)
.when(hopServer)
.buildSendExportMethod(
@@ -207,20 +200,12 @@ class HopServerTest {
HttpPost httpPostMock = mock(HttpPost.class);
URI uriMock = new URI("fake");
final String responseContent = "baah";
- when(httpPostMock.getURI()).thenReturn(uriMock);
- doReturn(uriMock).when(httpPostMock).getURI();
+ when(httpPostMock.getUri()).thenReturn(uriMock);
+ doReturn(uriMock).when(httpPostMock).getUri();
HttpClient client = mock(HttpClient.class);
- when(client.execute(any(), any(HttpContext.class)))
- .then(
- (Answer<HttpResponse>)
- invocation -> {
- HttpClientContext context = (HttpClientContext)
invocation.getArguments()[1];
- Credentials cred =
- context.getCredentialsProvider().getCredentials(new
AuthScope("hname", 1111));
- assertEquals("uname", cred.getUserPrincipal().getName());
- return mockResponse(200, responseContent);
- });
+ when(client.execute(any(ClassicHttpRequest.class), any(HttpContext.class)))
+ .then((Answer<ClassicHttpResponse>) invocation -> mockResponse(200,
responseContent));
// override init
when(hopServer.getHttpClient()).thenReturn(client);
when(hopServer.getResponseBodyAsString(any())).thenCallRealMethod();
@@ -248,15 +233,14 @@ class HopServerTest {
hopServer.setPort("" + port);
HttpClientContext auth = hopServer.getAuthContext(variables);
- Credentials cred = auth.getCredentialsProvider().getCredentials(new
AuthScope(host, port));
+ var cred = auth.getCredentialsProvider().getCredentials(new
AuthScope(host, port), null);
assertEquals(testUser, cred.getUserPrincipal().getName());
- assertEquals(testPassword, cred.getPassword());
String user2 = "user2";
hopServer.setUsername(user2);
hopServer.setPassword("pass2");
auth = hopServer.getAuthContext(variables);
- cred = auth.getCredentialsProvider().getCredentials(new AuthScope(host,
port));
+ cred = auth.getCredentialsProvider().getCredentials(new AuthScope(host,
port), null);
assertEquals(user2, cred.getUserPrincipal().getName());
}
@@ -269,8 +253,8 @@ class HopServerTest {
hopServer.setSslMode(true);
AuthCache cache = hopServer.getAuthContext(variables).getAuthCache();
- assertNotNull(cache.get(new HttpHost("localhost", 8443, "https")));
- assertNull(cache.get(new HttpHost("localhost", 8443, "http")));
+ assertNotNull(cache.get(new HttpHost("https", "localhost", 8443)));
+ assertNull(cache.get(new HttpHost("http", "localhost", 8443)));
}
@Test
@@ -282,8 +266,8 @@ class HopServerTest {
hopServer.setSslMode(false);
AuthCache cache = hopServer.getAuthContext(variables).getAuthCache();
- assertNull(cache.get(new HttpHost("localhost", 8080, "https")));
- assertNotNull(cache.get(new HttpHost("localhost", 8080, "http")));
+ assertNull(cache.get(new HttpHost("https", "localhost", 8080)));
+ assertNotNull(cache.get(new HttpHost("http", "localhost", 8080)));
}
@Test
diff --git a/lib/pom.xml b/lib/pom.xml
index 546f0b6694..ccfc8377d3 100644
--- a/lib/pom.xml
+++ b/lib/pom.xml
@@ -86,7 +86,6 @@
<guava.version>33.3.1-jre</guava.version>
<hive.version>2.3.9</hive.version>
<httpccore.version>4.4.15</httpccore.version>
- <httpclient.version>4.5.14</httpclient.version>
<httpclient5.version>5.6</httpclient5.version>
<jackcess.version>4.0.4</jackcess.version>
<jackson-annotations.version>2.21</jackson-annotations.version>
@@ -609,11 +608,6 @@
<artifactId>hop-ui</artifactId>
<version>${project.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <version>${httpclient.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
diff --git a/plugins/misc/rest/pom.xml b/plugins/misc/rest/pom.xml
index 52fca836a8..a503d8ee05 100644
--- a/plugins/misc/rest/pom.xml
+++ b/plugins/misc/rest/pom.xml
@@ -33,14 +33,14 @@
<dependencies>
<dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <version>4.5.14</version>
+ <groupId>org.apache.httpcomponents.client5</groupId>
+ <artifactId>httpclient5</artifactId>
+ <version>5.6</version>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.connectors</groupId>
- <artifactId>jersey-apache-connector</artifactId>
- <version>3.1.11</version>
+ <artifactId>jersey-apache5-connector</artifactId>
+ <version>4.0.2</version>
</dependency>
</dependencies>
diff --git a/plugins/misc/rest/src/assembly/assembly.xml
b/plugins/misc/rest/src/assembly/assembly.xml
index d9de5bb1d4..beb005abf4 100644
--- a/plugins/misc/rest/src/assembly/assembly.xml
+++ b/plugins/misc/rest/src/assembly/assembly.xml
@@ -49,7 +49,7 @@
</dependencySet>
<dependencySet>
<includes>
-
<include>org.glassfish.jersey.connectors:jersey-apache-connector:jar</include>
+
<include>org.glassfish.jersey.connectors:jersey-apache5-connector:jar</include>
</includes>
<outputDirectory>plugins/misc/rest/lib</outputDirectory>
</dependencySet>
diff --git
a/plugins/transforms/dorisbulkloader/src/main/java/org/apache/hop/pipeline/transforms/dorisbulkloader/DorisStreamLoad.java
b/plugins/transforms/dorisbulkloader/src/main/java/org/apache/hop/pipeline/transforms/dorisbulkloader/DorisStreamLoad.java
index a3664be5fb..bae52f60fb 100644
---
a/plugins/transforms/dorisbulkloader/src/main/java/org/apache/hop/pipeline/transforms/dorisbulkloader/DorisStreamLoad.java
+++
b/plugins/transforms/dorisbulkloader/src/main/java/org/apache/hop/pipeline/transforms/dorisbulkloader/DorisStreamLoad.java
@@ -24,17 +24,18 @@ import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.UUID;
import org.apache.commons.codec.binary.Base64;
+import org.apache.hc.client5.http.classic.methods.HttpPut;
+import org.apache.hc.client5.http.entity.GzipCompressingEntity;
+import org.apache.hc.client5.http.impl.DefaultRedirectStrategy;
+import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
+import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse;
+import org.apache.hc.client5.http.impl.classic.HttpClients;
+import org.apache.hc.core5.http.HttpHeaders;
+import org.apache.hc.core5.http.ParseException;
+import org.apache.hc.core5.http.io.entity.EntityUtils;
+import org.apache.hc.core5.http.io.entity.InputStreamEntity;
import org.apache.hop.core.encryption.Encr;
import org.apache.hop.core.json.HopJson;
-import org.apache.http.HttpHeaders;
-import org.apache.http.client.entity.GzipCompressingEntity;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpPut;
-import org.apache.http.entity.InputStreamEntity;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.DefaultRedirectStrategy;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.http.util.EntityUtils;
public class DorisStreamLoad {
private static final byte[] JSON_ARRAY_START =
@@ -168,8 +169,11 @@ public class DorisStreamLoad {
}
httpHeaders.forEach(put::setHeader);
- InputStreamEntity entity = new InputStreamEntity(recordStream,
recordStream.getWriteLength());
- entity.setChunked(false);
+ InputStreamEntity entity =
+ new InputStreamEntity(
+ recordStream,
+ recordStream.getWriteLength(),
+ org.apache.hc.core5.http.ContentType.APPLICATION_OCTET_STREAM);
if (LoadConstants.JSON.equals(format)) {
put.setEntity(entity);
@@ -183,23 +187,23 @@ public class DorisStreamLoad {
HttpClients.custom()
.setRedirectStrategy(
new DefaultRedirectStrategy() {
- @Override
- protected boolean isRedirectable(String method) {
- // If the connection target is FE, you need to deal with
307 redirect。
- return true;
- }
+ // If the connection target is FE, you need to deal with
307 redirect.
})
.build();
}
CloseableHttpResponse response = httpClient.execute(put);
- final int statusCode = response.getStatusLine().getStatusCode();
+ final int statusCode = response.getCode();
if (statusCode == 200 && response.getEntity() != null) {
- String loadResult = EntityUtils.toString(response.getEntity());
+ String loadResult;
+ try {
+ loadResult = EntityUtils.toString(response.getEntity());
+ } catch (ParseException e) {
+ throw new IOException("Unable to parse Doris stream load response", e);
+ }
return OBJECT_MAPPER.readValue(loadResult, ResponseContent.class);
} else {
- throw new DorisStreamLoadException(
- "stream load error: " + response.getStatusLine().toString());
+ throw new DorisStreamLoadException("stream load error: " +
response.getReasonPhrase());
}
}
diff --git
a/plugins/transforms/execsqlrow/src/main/resources/org/apache/hop/pipeline/transforms/execsqlrow/messages/messages_en_US.properties
b/plugins/transforms/execsqlrow/src/main/resources/org/apache/hop/pipeline/transforms/execsqlrow/messages/messages_en_US.properties
index 979ceeeee2..b5fd7fb0c7 100644
---
a/plugins/transforms/execsqlrow/src/main/resources/org/apache/hop/pipeline/transforms/execsqlrow/messages/messages_en_US.properties
+++
b/plugins/transforms/execsqlrow/src/main/resources/org/apache/hop/pipeline/transforms/execsqlrow/messages/messages_en_US.properties
@@ -58,12 +58,12 @@
ExecSqlRowMeta.Exception.UnableToLoadTransformMetaFromXML=Unable to load transfo
ExecSqlRowMeta.Injection.COMMIT_SIZE=The number of rows to commit at a time.
ExecSqlRowMeta.Injection.CONNECTION_NAME=The name of the database connection
to get table names from.
ExecSqlRowMeta.Injection.DELETE_STATS=The optional field in the stream that
contains the number of records that were deleted.
-ExecSqlRowMeta.Injection.INSERT_STATS=The optional field in the stream that
contains the number of records that were updated.
+ExecSqlRowMeta.Injection.INSERT_STATS=The optional field in the stream that
contains the number of records that were inserted.
ExecSqlRowMeta.Injection.OUTPUT_FIELDS=The output fields to write to.
ExecSqlRowMeta.Injection.READ_SQL_FROM_FILE=Specify whether the field name
will be SQL or the path to a SQL file.
ExecSqlRowMeta.Injection.READ_STATS=The optional field in the stream that
contains the number of records that were read.
ExecSqlRowMeta.Injection.SEND_SINGLE_STATEMENT=The option to group rows into a
single statement.
ExecSqlRowMeta.Injection.SQL_FIELD_NAME=The field that contains either the SQL
to execute or the path to a SQL file to execute.
-ExecSqlRowMeta.Injection.UPDATE_STATS=The optional field in the stream that
contains the number of records that were inserted.
+ExecSqlRowMeta.Injection.UPDATE_STATS=The optional field in the stream that
contains the number of records that were updated.
ExecSqlRowMeta.keyword=execute,sql,row
Update.Log.UnableToCommitUpdateConnection=Unable to commit connection \:
diff --git
a/plugins/transforms/execsqlrow/src/test/java/org/apache/hop/pipeline/transforms/execsqlrow/ExecSqlRowMetaTest.java
b/plugins/transforms/execsqlrow/src/test/java/org/apache/hop/pipeline/transforms/execsqlrow/ExecSqlRowMetaTest.java
index 9fbeb242d4..8a533f6dd4 100644
---
a/plugins/transforms/execsqlrow/src/test/java/org/apache/hop/pipeline/transforms/execsqlrow/ExecSqlRowMetaTest.java
+++
b/plugins/transforms/execsqlrow/src/test/java/org/apache/hop/pipeline/transforms/execsqlrow/ExecSqlRowMetaTest.java
@@ -16,14 +16,18 @@
*/
package org.apache.hop.pipeline.transforms.execsqlrow;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.hop.core.HopEnvironment;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.plugins.PluginRegistry;
import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
+import org.apache.hop.metadata.inject.HopMetadataInjector;
import org.apache.hop.pipeline.transforms.loadsave.LoadSaveTester;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -62,4 +66,19 @@ class ExecSqlRowMetaTest {
void testSerialization() throws HopException {
loadSaveTester.testSerialization();
}
+
+ /**
+ * Scalar stats fields (e.g. UPDATE_STATS) are not under any {@link
+ * HopMetadataProperty#injectionGroupKey()}; they only appear in {@link
+ * HopMetadataInjector#findInjectionGroupKeys(Class)} when part of a list
group. MetaInject maps
+ * such keys as simple properties even when the UI marks target_detail=true.
+ */
+ @Test
+ void updateStatsIsNotInAnyInjectionGroup() throws Exception {
+ Map<String, Set<String>> groupKeys =
+ HopMetadataInjector.findInjectionGroupKeys(ExecSqlRowMeta.class);
+ assertFalse(
+ groupKeys.values().stream()
+ .anyMatch(keys -> keys != null && keys.contains("UPDATE_STATS")));
+ }
}
diff --git
a/plugins/transforms/http/src/main/java/org/apache/hop/pipeline/transforms/http/Http.java
b/plugins/transforms/http/src/main/java/org/apache/hop/pipeline/transforms/http/Http.java
index f535ffd609..909383e27d 100644
---
a/plugins/transforms/http/src/main/java/org/apache/hop/pipeline/transforms/http/Http.java
+++
b/plugins/transforms/http/src/main/java/org/apache/hop/pipeline/transforms/http/Http.java
@@ -25,6 +25,23 @@ import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hc.client5.http.auth.AuthCache;
+import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
+import org.apache.hc.client5.http.classic.methods.HttpGet;
+import org.apache.hc.client5.http.impl.auth.BasicAuthCache;
+import org.apache.hc.client5.http.impl.auth.BasicScheme;
+import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
+import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse;
+import org.apache.hc.client5.http.protocol.HttpClientContext;
+import org.apache.hc.core5.http.Header;
+import org.apache.hc.core5.http.HttpEntity;
+import org.apache.hc.core5.http.HttpHost;
+import org.apache.hc.core5.http.NameValuePair;
+import org.apache.hc.core5.http.ParseException;
+import org.apache.hc.core5.http.io.entity.ByteArrayEntity;
+import org.apache.hc.core5.http.io.entity.EntityUtils;
+import org.apache.hc.core5.http.message.BasicNameValuePair;
+import org.apache.hc.core5.net.URIBuilder;
import org.apache.hop.core.Const;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.exception.HopTransformException;
@@ -39,22 +56,6 @@ import org.apache.hop.pipeline.Pipeline;
import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.BaseTransform;
import org.apache.hop.pipeline.transform.TransformMeta;
-import org.apache.http.Header;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpHost;
-import org.apache.http.HttpResponse;
-import org.apache.http.NameValuePair;
-import org.apache.http.client.AuthCache;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.protocol.HttpClientContext;
-import org.apache.http.client.utils.URIBuilder;
-import org.apache.http.entity.ByteArrayEntity;
-import org.apache.http.impl.auth.BasicScheme;
-import org.apache.http.impl.client.BasicAuthCache;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.message.BasicNameValuePair;
-import org.apache.http.util.EntityUtils;
import org.jetbrains.annotations.NotNull;
import org.json.simple.JSONObject;
@@ -126,13 +127,16 @@ public class Http extends BaseTransform<HttpMeta,
HttpData> {
// used for calculating the responseTime
long startTime = System.currentTimeMillis();
- // Preemptive authentication
- HttpHost target = new HttpHost(uri.getHost(), uri.getPort(),
uri.getScheme());
- if (StringUtils.isNotBlank(data.realProxyHost)) {
- target = new HttpHost(data.realProxyHost, data.realProxyPort,
"http");
- }
+ // Preemptive Basic auth: AuthCache must be keyed to the origin host
(same as the request
+ // target). Proxy routing is configured on the client via
RequestConfig — do not pass the
+ // proxy as HttpHost here (breaks credentials + preemptive cache
matching in HttpClient 5).
+ HttpHost target = HttpHost.create(uri);
- httpResponse = httpClient.execute(target, method,
buildtHttpClientContext(target));
+ httpResponse =
+ httpClient.execute(
+ target,
+ method,
+ buildHttpClientContext(target, data.realHttpLogin,
data.realHttpPassword));
// calculate the responseTime
long responseTime = System.currentTimeMillis() - startTime;
@@ -171,8 +175,6 @@ public class Http extends BaseTransform<HttpMeta, HttpData>
{
if (httpResponse != null) {
httpResponse.close();
}
- // Release current connection to the connection pool once you are done
- method.releaseConnection();
}
return newRow;
} catch (UnknownHostException uhe) {
@@ -231,28 +233,31 @@ public class Http extends BaseTransform<HttpMeta,
HttpData> {
byte[] bodyBytes = EntityUtils.toByteArray(entity);
dataVolumeIn = (dataVolumeIn != null ? dataVolumeIn : 0L) +
bodyBytes.length;
- ByteArrayEntity countedEntity = new ByteArrayEntity(bodyBytes);
- if (entity.getContentType() != null) {
- countedEntity.setContentType(entity.getContentType());
- }
- if (entity.getContentEncoding() != null) {
- countedEntity.setContentEncoding(entity.getContentEncoding());
- }
+ ByteArrayEntity countedEntity =
+ new ByteArrayEntity(
+ bodyBytes,
org.apache.hc.core5.http.ContentType.APPLICATION_OCTET_STREAM);
- return StringUtils.isEmpty(meta.getEncoding())
- ? EntityUtils.toString(countedEntity)
- : EntityUtils.toString(countedEntity, meta.getEncoding());
+ try {
+ return StringUtils.isEmpty(meta.getEncoding())
+ ? EntityUtils.toString(countedEntity)
+ : EntityUtils.toString(countedEntity, meta.getEncoding());
+ } catch (ParseException e) {
+ throw new IOException("Unable to parse HTTP response body", e);
+ }
}
- private static @NotNull HttpClientContext buildtHttpClientContext(HttpHost
target) {
- // Create AuthCache instance
+ /** HttpClient 5 requires {@link BasicScheme#initPreemptive} for preemptive
Basic auth. */
+ private static @NotNull HttpClientContext buildHttpClientContext(
+ HttpHost target, String httpLogin, String httpPassword) {
+ HttpClientContext localContext = HttpClientContext.create();
+ if (StringUtils.isBlank(httpLogin)) {
+ return localContext;
+ }
AuthCache authCache = new BasicAuthCache();
- // Generate BASIC scheme object and add it to the local
- // auth cache
BasicScheme basicAuth = new BasicScheme();
+ char[] passwordChars = httpPassword != null ? httpPassword.toCharArray() :
new char[0];
+ basicAuth.initPreemptive(new UsernamePasswordCredentials(httpLogin,
passwordChars));
authCache.put(target, basicAuth);
- // Add AuthCache to the execution context
- HttpClientContext localContext = HttpClientContext.create();
localContext.setAuthCache(authCache);
return localContext;
}
@@ -326,12 +331,12 @@ public class Http extends BaseTransform<HttpMeta,
HttpData> {
return uriBuilder;
}
- protected int requestStatusCode(HttpResponse httpResponse) {
- return httpResponse.getStatusLine().getStatusCode();
+ protected int requestStatusCode(CloseableHttpResponse httpResponse) {
+ return httpResponse.getCode();
}
protected Header[] searchForHeaders(CloseableHttpResponse response) {
- return response.getAllHeaders();
+ return response.getHeaders();
}
@Override
diff --git
a/plugins/transforms/http/src/main/java/org/apache/hop/pipeline/transforms/http/HttpData.java
b/plugins/transforms/http/src/main/java/org/apache/hop/pipeline/transforms/http/HttpData.java
index bb2802ae43..3eba2c5688 100644
---
a/plugins/transforms/http/src/main/java/org/apache/hop/pipeline/transforms/http/HttpData.java
+++
b/plugins/transforms/http/src/main/java/org/apache/hop/pipeline/transforms/http/HttpData.java
@@ -17,10 +17,10 @@
package org.apache.hop.pipeline.transforms.http;
+import org.apache.hc.core5.http.NameValuePair;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.pipeline.transform.BaseTransformData;
import org.apache.hop.pipeline.transform.ITransformData;
-import org.apache.http.NameValuePair;
@SuppressWarnings("java:S1104")
public class HttpData extends BaseTransformData implements ITransformData {
diff --git
a/plugins/transforms/http/src/test/java/org/apache/hop/pipeline/transforms/http/HttpDataTest.java
b/plugins/transforms/http/src/test/java/org/apache/hop/pipeline/transforms/http/HttpDataTest.java
index 91e0b37730..c30fc63cf6 100644
---
a/plugins/transforms/http/src/test/java/org/apache/hop/pipeline/transforms/http/HttpDataTest.java
+++
b/plugins/transforms/http/src/test/java/org/apache/hop/pipeline/transforms/http/HttpDataTest.java
@@ -23,9 +23,9 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import org.apache.hc.core5.http.NameValuePair;
+import org.apache.hc.core5.http.message.BasicNameValuePair;
import org.apache.hop.core.row.RowMeta;
-import org.apache.http.NameValuePair;
-import org.apache.http.message.BasicNameValuePair;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
diff --git
a/plugins/transforms/http/src/test/java/org/apache/hop/pipeline/transforms/http/HttpExecutionTest.java
b/plugins/transforms/http/src/test/java/org/apache/hop/pipeline/transforms/http/HttpExecutionTest.java
index 773f7d6fb5..579e81090e 100644
---
a/plugins/transforms/http/src/test/java/org/apache/hop/pipeline/transforms/http/HttpExecutionTest.java
+++
b/plugins/transforms/http/src/test/java/org/apache/hop/pipeline/transforms/http/HttpExecutionTest.java
@@ -29,6 +29,8 @@ import java.lang.reflect.Field;
import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
import java.util.concurrent.atomic.AtomicReference;
+import org.apache.hc.core5.http.NameValuePair;
+import org.apache.hc.core5.http.message.BasicNameValuePair;
import org.apache.hop.core.HopClientEnvironment;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.row.RowMeta;
@@ -37,7 +39,6 @@ import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.engines.local.LocalPipelineEngine;
import org.apache.hop.pipeline.transform.BaseTransform;
import org.apache.hop.pipeline.transform.TransformMeta;
-import org.apache.http.message.BasicNameValuePair;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
@@ -242,7 +243,7 @@ class HttpExecutionTest {
data.argNrs = new int[0];
data.useHeaderParameters = true;
data.headerParameters =
- new org.apache.http.NameValuePair[] {
+ new NameValuePair[] {
new BasicNameValuePair("X-Custom-Header", "" /* value comes from row
*/)
};
data.headerParametersNrs = new int[] {0};
diff --git
a/plugins/transforms/http/src/test/java/org/apache/hop/pipeline/transforms/http/HttpTest.java
b/plugins/transforms/http/src/test/java/org/apache/hop/pipeline/transforms/http/HttpTest.java
index e34f75ad59..cd9586e246 100644
---
a/plugins/transforms/http/src/test/java/org/apache/hop/pipeline/transforms/http/HttpTest.java
+++
b/plugins/transforms/http/src/test/java/org/apache/hop/pipeline/transforms/http/HttpTest.java
@@ -28,15 +28,15 @@ import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.HttpURLConnection;
import java.nio.charset.StandardCharsets;
+import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse;
+import org.apache.hc.core5.http.Header;
+import org.apache.hc.core5.http.io.entity.StringEntity;
+import org.apache.hc.core5.http.message.BasicHeader;
import org.apache.hop.core.exception.HopTransformException;
import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.engines.local.LocalPipelineEngine;
import org.apache.hop.pipeline.transform.BaseTransform;
import org.apache.hop.pipeline.transform.TransformMeta;
-import org.apache.http.Header;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.message.BasicHeader;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
diff --git a/plugins/transforms/httppost/pom.xml
b/plugins/transforms/httppost/pom.xml
index 2d5eefc4e9..e1da65b516 100644
--- a/plugins/transforms/httppost/pom.xml
+++ b/plugins/transforms/httppost/pom.xml
@@ -30,20 +30,14 @@
<name>Hop Plugins Transforms HTTP Post</name>
<properties>
- <httpclient.version>4.5.14</httpclient.version>
+ <httpclient.version>5.6</httpclient.version>
</properties>
<dependencies>
<dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpmime</artifactId>
+ <groupId>org.apache.httpcomponents.client5</groupId>
+ <artifactId>httpclient5</artifactId>
<version>${httpclient.version}</version>
- <exclusions>
- <exclusion>
- <groupId>*</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
</dependency>
</dependencies>
</project>
diff --git
a/plugins/transforms/httppost/src/main/java/org/apache/hop/pipeline/transforms/httppost/HttpPost.java
b/plugins/transforms/httppost/src/main/java/org/apache/hop/pipeline/transforms/httppost/HttpPost.java
index f6e0d2a551..16ed7cfcdd 100644
---
a/plugins/transforms/httppost/src/main/java/org/apache/hop/pipeline/transforms/httppost/HttpPost.java
+++
b/plugins/transforms/httppost/src/main/java/org/apache/hop/pipeline/transforms/httppost/HttpPost.java
@@ -31,6 +31,27 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hc.client5.http.auth.AuthCache;
+import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
+import org.apache.hc.client5.http.entity.UrlEncodedFormEntity;
+import org.apache.hc.client5.http.entity.mime.MultipartEntityBuilder;
+import org.apache.hc.client5.http.impl.auth.BasicAuthCache;
+import org.apache.hc.client5.http.impl.auth.BasicScheme;
+import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
+import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse;
+import org.apache.hc.client5.http.protocol.HttpClientContext;
+import org.apache.hc.core5.http.ContentType;
+import org.apache.hc.core5.http.Header;
+import org.apache.hc.core5.http.HttpEntity;
+import org.apache.hc.core5.http.HttpHost;
+import org.apache.hc.core5.http.NameValuePair;
+import org.apache.hc.core5.http.ParseException;
+import org.apache.hc.core5.http.io.entity.ByteArrayEntity;
+import org.apache.hc.core5.http.io.entity.EntityUtils;
+import org.apache.hc.core5.http.io.entity.StringEntity;
+import org.apache.hc.core5.http.message.BasicNameValuePair;
+import org.apache.hc.core5.net.URIBuilder;
+import org.apache.hc.core5.net.URLEncodedUtils;
import org.apache.hop.core.Const;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.exception.HopFileException;
@@ -47,26 +68,6 @@ import org.apache.hop.pipeline.Pipeline;
import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.BaseTransform;
import org.apache.hop.pipeline.transform.TransformMeta;
-import org.apache.http.Header;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpHost;
-import org.apache.http.HttpResponse;
-import org.apache.http.NameValuePair;
-import org.apache.http.client.AuthCache;
-import org.apache.http.client.entity.UrlEncodedFormEntity;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.protocol.HttpClientContext;
-import org.apache.http.client.utils.URIBuilder;
-import org.apache.http.client.utils.URLEncodedUtils;
-import org.apache.http.entity.ByteArrayEntity;
-import org.apache.http.entity.ContentType;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.entity.mime.MultipartEntityBuilder;
-import org.apache.http.impl.auth.BasicScheme;
-import org.apache.http.impl.client.BasicAuthCache;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.message.BasicNameValuePair;
-import org.apache.http.util.EntityUtils;
import org.json.simple.JSONObject;
/** Make a HTTP Post call */
@@ -123,8 +124,8 @@ public class HttpPost extends BaseTransform<HttpPostMeta,
HttpPostData> {
}
URIBuilder uriBuilder = new URIBuilder(data.realUrl);
URI uri = uriBuilder.build();
- org.apache.http.client.methods.HttpPost post =
- new org.apache.http.client.methods.HttpPost(uri);
+ org.apache.hc.client5.http.classic.methods.HttpPost post =
+ new org.apache.hc.client5.http.classic.methods.HttpPost(uri);
MultipartEntityBuilder multipart = null;
boolean useMultipart = meta.isPostAFile() || meta.isMultipartupload();
@@ -153,21 +154,20 @@ public class HttpPost extends BaseTransform<HttpPostMeta,
HttpPostData> {
// used for calculating the responseTime
long startTime = System.currentTimeMillis();
- // Execute the POST method
- HttpHost target = new HttpHost(uri.getHost(), uri.getPort(),
uri.getScheme());
- if (StringUtils.isNotBlank(data.realProxyHost)) {
- target = new HttpHost(data.realProxyHost, data.realProxyPort,
"http");
- }
+ // Origin host for routing + preemptive Basic auth cache (proxy is on
the client, not here).
+ HttpHost target = HttpHost.create(uri);
- // Create AuthCache instance
- AuthCache authCache = new BasicAuthCache();
- // Generate BASIC scheme object and add it to the local
- // auth cache
- BasicScheme basicAuth = new BasicScheme();
- authCache.put(target, basicAuth);
- // Add AuthCache to the execution context
HttpClientContext localContext = HttpClientContext.create();
- localContext.setAuthCache(authCache);
+ if (StringUtils.isNotBlank(data.realHttpLogin)) {
+ AuthCache authCache = new BasicAuthCache();
+ BasicScheme basicAuth = new BasicScheme();
+ char[] passwordChars =
+ data.realHttpPassword != null ?
data.realHttpPassword.toCharArray() : new char[0];
+ basicAuth.initPreemptive(
+ new UsernamePasswordCredentials(data.realHttpLogin,
passwordChars));
+ authCache.put(target, basicAuth);
+ localContext.setAuthCache(authCache);
+ }
httpResponse = httpClient.execute(target, post, localContext);
@@ -248,8 +248,6 @@ public class HttpPost extends BaseTransform<HttpPostMeta,
HttpPostData> {
newRow = RowDataUtil.addValueData(newRow, returnFieldsOffset,
headerString);
}
} finally {
- // Release current connection to the connection pool once you are done
- post.releaseConnection();
if (httpResponse != null) {
httpResponse.close();
}
@@ -292,22 +290,20 @@ public class HttpPost extends BaseTransform<HttpPostMeta,
HttpPostData> {
byte[] bodyBytes = EntityUtils.toByteArray(entity);
dataVolumeIn = (dataVolumeIn != null ? dataVolumeIn : 0L) +
bodyBytes.length;
- ByteArrayEntity countedEntity = new ByteArrayEntity(bodyBytes);
- if (entity.getContentType() != null) {
- countedEntity.setContentType(entity.getContentType());
- }
- if (entity.getContentEncoding() != null) {
- countedEntity.setContentEncoding(entity.getContentEncoding());
+ ByteArrayEntity countedEntity =
+ new ByteArrayEntity(bodyBytes, ContentType.APPLICATION_OCTET_STREAM);
+ try {
+ return EntityUtils.toString(countedEntity);
+ } catch (ParseException e) {
+ throw new HopException("Unable to parse HTTP response body", e);
}
-
- return EntityUtils.toString(countedEntity);
}
- protected int requestStatusCode(HttpResponse httpResponse) {
- return httpResponse.getStatusLine().getStatusCode();
+ protected int requestStatusCode(CloseableHttpResponse httpResponse) {
+ return httpResponse.getCode();
}
- protected InputStreamReader openStream(String encoding, HttpResponse
httpResponse)
+ protected InputStreamReader openStream(String encoding,
CloseableHttpResponse httpResponse)
throws Exception {
if (!Utils.isEmpty(encoding)) {
return new InputStreamReader(httpResponse.getEntity().getContent(),
encoding);
@@ -316,8 +312,8 @@ public class HttpPost extends BaseTransform<HttpPostMeta,
HttpPostData> {
}
}
- protected Header[] searchForHeaders(HttpResponse response) {
- return response.getAllHeaders();
+ protected Header[] searchForHeaders(CloseableHttpResponse response) {
+ return response.getHeaders();
}
@Override
@@ -519,7 +515,8 @@ public class HttpPost extends BaseTransform<HttpPostMeta,
HttpPostData> {
*
* @param post the HTTP POST request to which the multipart entity will be
attached
*/
- private void addHttpHeaders(org.apache.http.client.methods.HttpPost post,
Object[] rowData)
+ private void addHttpHeaders(
+ org.apache.hc.client5.http.classic.methods.HttpPost post, Object[]
rowData)
throws HopValueException {
// add HttpPost header ContentType
addHeadersContentType(post);
@@ -549,7 +546,7 @@ public class HttpPost extends BaseTransform<HttpPostMeta,
HttpPostData> {
*
* @param post the HTTP POST request to which the multipart entity will be
attached
*/
- private void addHeadersContentType(org.apache.http.client.methods.HttpPost
post) {
+ private void
addHeadersContentType(org.apache.hc.client5.http.classic.methods.HttpPost post)
{
// Specify content type and encoding
// If content encoding is not explicitly specified
// ISO-8859-1 is assumed by the POSTMethod
@@ -581,7 +578,8 @@ public class HttpPost extends BaseTransform<HttpPostMeta,
HttpPostData> {
* @param post the HTTP POST request to which the multipart entity will be
attached
* @param rowData row data
*/
- private void addQueryParams(org.apache.http.client.methods.HttpPost post,
Object[] rowData)
+ private void addQueryParams(
+ org.apache.hc.client5.http.classic.methods.HttpPost post, Object[]
rowData)
throws HopValueException, UnsupportedEncodingException {
if (!data.useQueryParameters) {
return;
@@ -608,7 +606,7 @@ public class HttpPost extends BaseTransform<HttpPostMeta,
HttpPostData> {
* @param rowData row data
*/
private void addBodyParams(
- org.apache.http.client.methods.HttpPost post,
+ org.apache.hc.client5.http.classic.methods.HttpPost post,
Object[] rowData,
MultipartEntityBuilder multipart)
throws HopException {
@@ -649,7 +647,7 @@ public class HttpPost extends BaseTransform<HttpPostMeta,
HttpPostData> {
* @param rowData row data
*/
private void addBodyFileParam(
- org.apache.http.client.methods.HttpPost post,
+ org.apache.hc.client5.http.classic.methods.HttpPost post,
Object[] rowData,
MultipartEntityBuilder multipart)
throws UnsupportedEncodingException, HopFileException, HopValueException
{
@@ -689,11 +687,12 @@ public class HttpPost extends BaseTransform<HttpPostMeta,
HttpPostData> {
}
@VisibleForTesting
- void attachRawRequestEntityIfNeeded(org.apache.http.client.methods.HttpPost
post, byte[] bytes) {
+ void attachRawRequestEntityIfNeeded(
+ org.apache.hc.client5.http.classic.methods.HttpPost post, byte[] bytes) {
// In the non-multipart case the request entity field is the raw POST body
when no other body
// has been configured.
if (post.getEntity() == null) {
- post.setEntity(new ByteArrayEntity(bytes));
+ post.setEntity(new ByteArrayEntity(bytes,
ContentType.APPLICATION_OCTET_STREAM));
}
}
@@ -709,7 +708,7 @@ public class HttpPost extends BaseTransform<HttpPostMeta,
HttpPostData> {
* @param multipart if {@code null}, no entity will be set on the request
*/
private void addBodyParamsAfter(
- org.apache.http.client.methods.HttpPost post, MultipartEntityBuilder
multipart) {
+ org.apache.hc.client5.http.classic.methods.HttpPost post,
MultipartEntityBuilder multipart) {
if (multipart != null) {
post.setEntity(multipart.build());
}
diff --git
a/plugins/transforms/httppost/src/main/java/org/apache/hop/pipeline/transforms/httppost/HttpPostData.java
b/plugins/transforms/httppost/src/main/java/org/apache/hop/pipeline/transforms/httppost/HttpPostData.java
index d40b247aad..7e46ee26bb 100644
---
a/plugins/transforms/httppost/src/main/java/org/apache/hop/pipeline/transforms/httppost/HttpPostData.java
+++
b/plugins/transforms/httppost/src/main/java/org/apache/hop/pipeline/transforms/httppost/HttpPostData.java
@@ -17,10 +17,10 @@
package org.apache.hop.pipeline.transforms.httppost;
+import org.apache.hc.core5.http.NameValuePair;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.pipeline.transform.BaseTransformData;
import org.apache.hop.pipeline.transform.ITransformData;
-import org.apache.http.NameValuePair;
@SuppressWarnings("java:S1104")
public class HttpPostData extends BaseTransformData implements ITransformData {
diff --git
a/plugins/transforms/httppost/src/test/java/org/apache/hop/pipeline/transforms/httppost/HttpPostExecutionTest.java
b/plugins/transforms/httppost/src/test/java/org/apache/hop/pipeline/transforms/httppost/HttpPostExecutionTest.java
index 0a608c686d..9638c5157b 100644
---
a/plugins/transforms/httppost/src/test/java/org/apache/hop/pipeline/transforms/httppost/HttpPostExecutionTest.java
+++
b/plugins/transforms/httppost/src/test/java/org/apache/hop/pipeline/transforms/httppost/HttpPostExecutionTest.java
@@ -29,6 +29,8 @@ import java.lang.reflect.Field;
import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
import java.util.concurrent.atomic.AtomicReference;
+import org.apache.hc.core5.http.NameValuePair;
+import org.apache.hc.core5.http.message.BasicNameValuePair;
import org.apache.hop.core.HopClientEnvironment;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.row.RowMeta;
@@ -37,7 +39,6 @@ import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.engines.local.LocalPipelineEngine;
import org.apache.hop.pipeline.transform.BaseTransform;
import org.apache.hop.pipeline.transform.TransformMeta;
-import org.apache.http.message.BasicNameValuePair;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
@@ -204,7 +205,7 @@ class HttpPostExecutionTest {
data.useBodyParameters = true;
data.body_parameters_nrs = new int[] {0};
data.bodyParameters =
- new org.apache.http.NameValuePair[] {
+ new NameValuePair[] {
new BasicNameValuePair("city", "") // name fixed; value comes from
row
};
RowMeta rowMeta = new RowMeta();
@@ -226,7 +227,7 @@ class HttpPostExecutionTest {
HttpPostData data = minimalData(baseUrl() + "/echo");
data.useQueryParameters = true;
data.query_parameters_nrs = new int[] {0};
- data.queryParameters = new org.apache.http.NameValuePair[] {new
BasicNameValuePair("q", "")};
+ data.queryParameters = new NameValuePair[] {new BasicNameValuePair("q",
"")};
RowMeta rowMeta = new RowMeta();
rowMeta.addValueMeta(new ValueMetaString("qField"));
data.inputRowMeta = rowMeta;
diff --git
a/plugins/transforms/httppost/src/test/java/org/apache/hop/pipeline/transforms/httppost/HttpPostTest.java
b/plugins/transforms/httppost/src/test/java/org/apache/hop/pipeline/transforms/httppost/HttpPostTest.java
index 06e648db1b..efbe75c7dd 100644
---
a/plugins/transforms/httppost/src/test/java/org/apache/hop/pipeline/transforms/httppost/HttpPostTest.java
+++
b/plugins/transforms/httppost/src/test/java/org/apache/hop/pipeline/transforms/httppost/HttpPostTest.java
@@ -25,19 +25,15 @@ import java.io.InputStreamReader;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.nio.charset.StandardCharsets;
+import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse;
+import org.apache.hc.core5.http.NameValuePair;
+import org.apache.hc.core5.http.io.entity.InputStreamEntity;
+import org.apache.hc.core5.http.io.entity.StringEntity;
+import org.apache.hc.core5.http.message.BasicNameValuePair;
import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.engines.local.LocalPipelineEngine;
import org.apache.hop.pipeline.transform.BaseTransform;
import org.apache.hop.pipeline.transform.TransformMeta;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpResponse;
-import org.apache.http.HttpVersion;
-import org.apache.http.NameValuePair;
-import org.apache.http.entity.InputStreamEntity;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.message.BasicHttpResponse;
-import org.apache.http.message.BasicNameValuePair;
-import org.apache.http.message.BasicStatusLine;
import org.junit.jupiter.api.Test;
class HttpPostTest {
@@ -84,8 +80,8 @@ class HttpPostTest {
void attachRawRequestEntityIfNeededSetsEntityWhenMissing() {
HttpPost http = newTransform();
- org.apache.http.client.methods.HttpPost post =
- new org.apache.http.client.methods.HttpPost("http://localhost";);
+ org.apache.hc.client5.http.classic.methods.HttpPost post =
+ new
org.apache.hc.client5.http.classic.methods.HttpPost("http://localhost";);
http.attachRawRequestEntityIfNeeded(post, "payload".getBytes());
@@ -96,9 +92,9 @@ class HttpPostTest {
void attachRawRequestEntityIfNeededDoesNotOverrideExistingEntity() throws
Exception {
HttpPost http = newTransform();
- org.apache.http.client.methods.HttpPost post =
- new org.apache.http.client.methods.HttpPost("http://localhost";);
- HttpEntity existing = new StringEntity("existing");
+ org.apache.hc.client5.http.classic.methods.HttpPost post =
+ new
org.apache.hc.client5.http.classic.methods.HttpPost("http://localhost";);
+ org.apache.hc.core5.http.HttpEntity existing = new
StringEntity("existing");
post.setEntity(existing);
http.attachRawRequestEntityIfNeeded(post, "payload".getBytes());
@@ -110,7 +106,8 @@ class HttpPostTest {
void collectRequestBytesCountsUnknownLengthEntity() {
HttpPost http = newTransform();
- HttpEntity entity = new InputStreamEntity(new
ByteArrayInputStream("payload".getBytes()), -1);
+ org.apache.hc.core5.http.HttpEntity entity =
+ new InputStreamEntity(new ByteArrayInputStream("payload".getBytes()),
-1, null);
http.collectRequestBytes(entity);
@@ -121,7 +118,8 @@ class HttpPostTest {
void collectRequestBytesCountsKnownLengthEntity() {
HttpPost http = newTransform();
- HttpEntity entity = new StringEntity("payload", StandardCharsets.UTF_8);
+ org.apache.hc.core5.http.HttpEntity entity =
+ new StringEntity("payload", StandardCharsets.UTF_8);
http.collectRequestBytes(entity);
@@ -144,7 +142,7 @@ class HttpPostTest {
@Test
void openStreamWithEncodingReturnsReaderDecodedInThatEncoding() throws
Exception {
HttpPost http = newTransform();
- HttpResponse response = responseWithBody("hello");
+ CloseableHttpResponse response = responseWithBody("hello");
try (InputStreamReader reader = http.openStream("UTF-8", response)) {
char[] buf = new char[5];
@@ -157,7 +155,7 @@ class HttpPostTest {
@Test
void openStreamWithoutEncodingReturnsDefaultReader() throws Exception {
HttpPost http = newTransform();
- HttpResponse response = responseWithBody("world");
+ CloseableHttpResponse response = responseWithBody("world");
try (InputStreamReader reader = http.openStream(null, response)) {
char[] buf = new char[5];
@@ -170,7 +168,7 @@ class HttpPostTest {
@Test
void openStreamWithEmptyEncodingReturnsDefaultReader() throws Exception {
HttpPost http = newTransform();
- HttpResponse response = responseWithBody("hop");
+ CloseableHttpResponse response = responseWithBody("hop");
try (InputStreamReader reader = http.openStream("", response)) {
char[] buf = new char[3];
@@ -180,16 +178,17 @@ class HttpPostTest {
}
}
- private static HttpResponse responseWithBody(String body) {
- BasicHttpResponse response =
- new BasicHttpResponse(new BasicStatusLine(HttpVersion.HTTP_1_1, 200,
"OK"));
- response.setEntity(new StringEntity(body, StandardCharsets.UTF_8));
+ private static CloseableHttpResponse responseWithBody(String body) {
+ CloseableHttpResponse response =
org.mockito.Mockito.mock(CloseableHttpResponse.class);
+ org.mockito.Mockito.when(response.getEntity())
+ .thenReturn(new StringEntity(body, StandardCharsets.UTF_8));
return response;
}
private static Object invokePrivate(Object target, String methodName,
Object... args)
throws Exception {
- Method method = target.getClass().getDeclaredMethod(methodName,
HttpEntity.class);
+ Method method =
+ target.getClass().getDeclaredMethod(methodName,
org.apache.hc.core5.http.HttpEntity.class);
method.setAccessible(true);
return method.invoke(target, args);
}
diff --git
a/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInject.java
b/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInject.java
index fcd84c4f3d..c348deb162 100644
---
a/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInject.java
+++
b/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInject.java
@@ -352,7 +352,8 @@ public class MetaInject extends
BaseTransform<MetaInjectMeta, MetaInjectData> {
groupRowMetaMap,
groupSourceMap,
injectionGroupData,
- injectionKeyData);
+ injectionKeyData,
+ targetTransformMeta.getClass());
}
// Now all source rows are mapped and we can construct the group data map
@@ -387,8 +388,9 @@ public class MetaInject extends
BaseTransform<MetaInjectMeta, MetaInjectData> {
Map<String, IRowMeta> groupRowMetaMap,
Map<String, String> groupSourceMap,
Map<String, RowBuffer> injectionGroupData,
- Map<String, Object> injectionKeyData)
- throws HopTransformException, HopValueException {
+ Map<String, Object> injectionKeyData,
+ Class<?> targetMetaClass)
+ throws HopException {
if
(!targetTransformName.equalsIgnoreCase(mapping.getTargetTransformName())) {
// Covered elsewhere.
return;
@@ -398,16 +400,7 @@ public class MetaInject extends
BaseTransform<MetaInjectMeta, MetaInjectData> {
// To which group does this belong?
//
String groupKey = lookupGroupKey(injectionKeyGroupMap,
mapping.getTargetAttributeKey());
- if (mapping.isTargetDetail() || groupKey != null) {
- // If it's a detail and not part of a group, we have a configuration
error.
- //
- if (groupKey == null) {
- throw new HopTransformException(
- "The injection group key for target key '"
- + mapping.getTargetAttributeKey()
- + "' was not found.");
- }
-
+ if (groupKey != null) {
if (sourceRows != null && !sourceRows.isEmpty()) {
// We need to collect the data for the mapping
//
@@ -420,6 +413,19 @@ public class MetaInject extends
BaseTransform<MetaInjectMeta, MetaInjectData> {
addConstantToGroupData(mapping, injectionGroupData, groupKey);
}
}
+ } else if (mapping.isTargetDetail()) {
+ // Detail flag is often used for grouped (list) keys. Scalar
HopMetadataProperty keys (e.g.
+ // ExecSqlRow UPDATE_STATS) are not part of any injection group; treat
them as simple keys.
+ //
+ if (HopMetadataInjector.isTopLevelInjectionKey(
+ targetMetaClass, mapping.getTargetAttributeKey())) {
+ collectInjectionKeyValue(mapping, sourceRows, injectionKeyData);
+ } else {
+ throw new HopTransformException(
+ "The injection group key for target key '"
+ + mapping.getTargetAttributeKey()
+ + "' was not found.");
+ }
} else {
// A simple property from the first row.
// This also captures the "Constant" mappings where we don't have source
rows to feed the
diff --git
a/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectDialog.java
b/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectDialog.java
index ea2384e96d..db14e428df 100644
---
a/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectDialog.java
+++
b/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectDialog.java
@@ -933,6 +933,26 @@ public class MetaInjectDialog extends BaseTransformDialog {
* @param metaInterface The transform ITransformMeta
* @return true if there was at least one used key
*/
+ /**
+ * Match a tree row to a saved mapping by target transform and injection key
only. {@link
+ * MetaInjectMapping#equals} also compares {@code targetDetail}; older
pipelines may store {@code
+ * target_detail=Y} for scalar {@link
org.apache.hop.metadata.api.HopMetadataProperty} keys that
+ * the UI places in the root group ({@code targetDetail=N}), which would
otherwise hide sources in
+ * the tree.
+ */
+ private static MetaInjectMapping findMappingForTargetKey(
+ List<MetaInjectMapping> mappings, MetaInjectMapping template) {
+ for (MetaInjectMapping m : mappings) {
+ if (Const.NVL(m.getTargetTransformName(), "")
+ .equalsIgnoreCase(Const.NVL(template.getTargetTransformName(),
""))
+ && Const.NVL(m.getTargetAttributeKey(), "")
+ .equalsIgnoreCase(Const.NVL(template.getTargetAttributeKey(),
""))) {
+ return m;
+ }
+ }
+ return null;
+ }
+
private boolean processMDIDescription(
TransformMeta transformMeta, TreeItem transformItem, ITransformMeta
metaInterface) {
boolean hasUsedKeys = false;
@@ -973,11 +993,9 @@ public class MetaInjectDialog extends BaseTransformDialog {
treeItemTargetMap.put(treeItem, targetMapping);
- // Equals only works on the target fields
- int index = targetMappings.indexOf(targetMapping);
- if (index >= 0) {
- MetaInjectMapping mapping = targetMappings.get(index);
- // Update with source mapping information
+ MetaInjectMapping mapping = findMappingForTargetKey(targetMappings,
targetMapping);
+ if (mapping != null) {
+ mapping.setTargetDetail(targetMapping.isTargetDetail());
treeItemTargetMap.put(treeItem, mapping);
hasUsedKeys = true;
treeItem.setText(
diff --git a/plugins/transforms/rest/pom.xml b/plugins/transforms/rest/pom.xml
index 74719d42c0..a14fc99acf 100644
--- a/plugins/transforms/rest/pom.xml
+++ b/plugins/transforms/rest/pom.xml
@@ -37,8 +37,8 @@
</dependency>
<dependency>
<groupId>org.glassfish.jersey.connectors</groupId>
- <artifactId>jersey-apache-connector</artifactId>
- <version>3.1.11</version>
+ <artifactId>jersey-apache5-connector</artifactId>
+ <version>4.0.2</version>
</dependency>
</dependencies>
diff --git a/plugins/transforms/rest/src/assembly/assembly.xml
b/plugins/transforms/rest/src/assembly/assembly.xml
index fc990d0ca8..e0baa647c8 100644
--- a/plugins/transforms/rest/src/assembly/assembly.xml
+++ b/plugins/transforms/rest/src/assembly/assembly.xml
@@ -52,7 +52,7 @@
</dependencySet>
<dependencySet>
<includes>
-
<include>org.glassfish.jersey.connectors:jersey-apache-connector:jar</include>
+
<include>org.glassfish.jersey.connectors:jersey-apache5-connector:jar</include>
</includes>
<outputDirectory>plugins/transforms/rest/lib</outputDirectory>
</dependencySet>
diff --git
a/plugins/transforms/rest/src/main/java/org/apache/hop/pipeline/transforms/rest/Rest.java
b/plugins/transforms/rest/src/main/java/org/apache/hop/pipeline/transforms/rest/Rest.java
index 37ddcd4ce5..d4bab044f7 100644
---
a/plugins/transforms/rest/src/main/java/org/apache/hop/pipeline/transforms/rest/Rest.java
+++
b/plugins/transforms/rest/src/main/java/org/apache/hop/pipeline/transforms/rest/Rest.java
@@ -59,7 +59,7 @@ import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.BaseTransform;
import org.apache.hop.pipeline.transform.TransformMeta;
import org.apache.hop.pipeline.transforms.rest.common.RestConst;
-import org.glassfish.jersey.apache.connector.ApacheConnectorProvider;
+import org.glassfish.jersey.apache5.connector.Apache5ConnectorProvider;
import org.glassfish.jersey.client.ClientConfig;
import org.glassfish.jersey.client.ClientProperties;
import org.glassfish.jersey.client.HttpUrlConnectorProvider;
@@ -633,7 +633,7 @@ public class Rest extends BaseTransform<RestMeta, RestData>
{
if (data.config == null) {
// Use ApacheHttpClient for supporting proxy authentication.
data.config = new ClientConfig();
- data.config.connectorProvider(new ApacheConnectorProvider());
+ data.config.connectorProvider(new Apache5ConnectorProvider());
data.config.property(
ClientProperties.REQUEST_ENTITY_PROCESSING,
RequestEntityProcessing.BUFFERED);
data.config.property(ClientProperties.SUPPRESS_HTTP_COMPLIANCE_VALIDATION,
true);
diff --git
a/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRows.java
b/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRows.java
index 2cfe9bb03a..bcbefe45f9 100644
---
a/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRows.java
+++
b/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRows.java
@@ -464,6 +464,23 @@ public class SortRows extends BaseTransform<SortRowsMeta,
SortRowsData> {
toConvert.add(data.fieldnrs[i]);
}
}
+ // Batch boundaries for "group sort" (merge-join style): only fields
marked presorted in the
+ // meta participate. When none are presorted, groupnrs is empty and
sameGroup() is always true
+ // so all rows accumulate for one global sort (see commit 36af1f6f
regression when this used all
+ // sort keys).
+ List<Integer> groupIndices = new ArrayList<>();
+ for (SortRowsField sortField : meta.getSortFields()) {
+ if (sortField.isPreSortedField()) {
+ int idx = inputRowMeta.indexOfValue(sortField.getFieldName());
+ if (idx < 0) {
+ throw new HopException(
+ BaseMessages.getString(
+ PKG, "SortRows.Error.PresortedFieldNotFound",
sortField.getFieldName()));
+ }
+ groupIndices.add(idx);
+ }
+ }
+ data.groupnrs =
groupIndices.stream().mapToInt(Integer::intValue).toArray();
data.convertKeysToNative = toConvert.isEmpty() ? null : new
int[toConvert.size()];
int i = 0;
@@ -472,6 +489,8 @@ public class SortRows extends BaseTransform<SortRowsMeta,
SortRowsData> {
i++;
}
data.rowComparator = new RowObjectArrayComparator(data.outputRowMeta,
data.fieldnrs);
+ // Ensure first incoming row initializes "previous" before sameGroup() is
evaluated.
+ data.newBatch = true;
return false;
}
@@ -656,10 +675,11 @@ public class SortRows extends BaseTransform<SortRowsMeta,
SortRowsData> {
*/
// Is the row r of the same group as previous?
private boolean sameGroup(Object[] previous, Object[] r) throws
HopValueException {
- if (r == null) {
+ if (previous == null || r == null) {
return false;
}
- return getInputRowMeta().compare(previous, r, data.groupnrs) == 0;
+ int[] groupFields = data.groupnrs != null ? data.groupnrs : data.fieldnrs;
+ return groupFields != null && getInputRowMeta().compare(previous, r,
groupFields) == 0;
}
private void setPrevious(Object[] r) throws HopException {
diff --git
a/plugins/transforms/sort/src/test/java/org/apache/hop/pipeline/transforms/sort/SortRowsTest.java
b/plugins/transforms/sort/src/test/java/org/apache/hop/pipeline/transforms/sort/SortRowsTest.java
new file mode 100644
index 0000000000..5cec637a2e
--- /dev/null
+++
b/plugins/transforms/sort/src/test/java/org/apache/hop/pipeline/transforms/sort/SortRowsTest.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.sort;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.when;
+
+import java.lang.reflect.Method;
+import java.util.Collections;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.pipeline.Pipeline;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+class SortRowsTest {
+
+ @Test
+ void processRowFirstCallInitializesGroupFieldsAndBatchState() throws
Exception {
+ SortRowsMeta meta = new SortRowsMeta();
+ meta.setSortFields(
+ Collections.singletonList(new SortRowsField("id", true, true, false,
0, false)));
+ SortRowsData data = new SortRowsData();
+ SortRows sortRows = newSortRows(meta, data);
+
+ RowMeta inputRowMeta = new RowMeta();
+ inputRowMeta.addValueMeta(new ValueMetaString("id"));
+ sortRows.setInputRowMeta(inputRowMeta);
+
+ Object[] firstRow = new Object[] {"1"};
+
+ Method processRowFirstCall =
+ SortRows.class.getDeclaredMethod("processRowFirstCall",
Object[].class);
+ processRowFirstCall.setAccessible(true);
+ boolean done = (boolean) processRowFirstCall.invoke(sortRows, (Object)
firstRow);
+
+ assertFalse(done);
+ assertArrayEquals(new int[0], data.groupnrs);
+ assertTrue(data.newBatch);
+ }
+
+ @Test
+ void processRowFirstCallUsesPresortedFieldsForGroupIndices() throws
Exception {
+ SortRowsMeta meta = new SortRowsMeta();
+ meta.setSortFields(
+ Collections.singletonList(new SortRowsField("id", true, true, false,
0, true)));
+ SortRowsData data = new SortRowsData();
+ SortRows sortRows = newSortRows(meta, data);
+
+ RowMeta inputRowMeta = new RowMeta();
+ inputRowMeta.addValueMeta(new ValueMetaString("id"));
+ sortRows.setInputRowMeta(inputRowMeta);
+
+ Method processRowFirstCall =
+ SortRows.class.getDeclaredMethod("processRowFirstCall",
Object[].class);
+ processRowFirstCall.setAccessible(true);
+ processRowFirstCall.invoke(sortRows, (Object) new Object[] {"1"});
+
+ assertArrayEquals(new int[] {0}, data.groupnrs);
+ }
+
+ @Test
+ void sameGroupReturnsFalseWhenPreviousRowIsNull() throws Exception {
+ SortRowsMeta meta = new SortRowsMeta();
+ SortRowsData data = new SortRowsData();
+ data.fieldnrs = new int[] {0};
+ data.groupnrs = new int[] {0};
+ SortRows sortRows = newSortRows(meta, data);
+
+ RowMeta inputRowMeta = new RowMeta();
+ inputRowMeta.addValueMeta(new ValueMetaString("id"));
+ sortRows.setInputRowMeta(inputRowMeta);
+
+ Method sameGroup =
+ SortRows.class.getDeclaredMethod("sameGroup", Object[].class,
Object[].class);
+ sameGroup.setAccessible(true);
+
+ Object[] row = new Object[] {"1"};
+ assertDoesNotThrow(() -> sameGroup.invoke(sortRows, null, row));
+ boolean result = (boolean) sameGroup.invoke(sortRows, null, row);
+ assertFalse(result);
+ }
+
+ private SortRows newSortRows(SortRowsMeta meta, SortRowsData data) {
+ TransformMeta transformMeta = new TransformMeta();
+ transformMeta.setName("SortRows");
+ Pipeline pipeline = Mockito.mock(Pipeline.class);
+ when(pipeline.getVariableNames()).thenReturn(new String[0]);
+ PipelineMeta pipelineMeta = new PipelineMeta();
+ pipelineMeta.addTransform(transformMeta);
+ return new SortRows(transformMeta, meta, data, 0, pipelineMeta, pipeline);
+ }
+}
diff --git
a/plugins/transforms/webservices/src/main/java/org/apache/hop/pipeline/transforms/webservices/WebService.java
b/plugins/transforms/webservices/src/main/java/org/apache/hop/pipeline/transforms/webservices/WebService.java
index 623334d574..5febe7d5ba 100644
---
a/plugins/transforms/webservices/src/main/java/org/apache/hop/pipeline/transforms/webservices/WebService.java
+++
b/plugins/transforms/webservices/src/main/java/org/apache/hop/pipeline/transforms/webservices/WebService.java
@@ -23,6 +23,7 @@ import java.io.InputStreamReader;
import java.io.StringReader;
import java.io.StringWriter;
import java.math.BigDecimal;
+import java.net.URI;
import java.net.URISyntaxException;
import java.net.UnknownHostException;
import java.nio.charset.Charset;
@@ -49,6 +50,22 @@ import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hc.client5.http.auth.AuthCache;
+import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
+import org.apache.hc.client5.http.classic.HttpClient;
+import org.apache.hc.client5.http.classic.methods.HttpPost;
+import org.apache.hc.client5.http.impl.auth.BasicAuthCache;
+import org.apache.hc.client5.http.impl.auth.BasicScheme;
+import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
+import org.apache.hc.client5.http.protocol.HttpClientContext;
+import org.apache.hc.core5.http.ContentType;
+import org.apache.hc.core5.http.Header;
+import org.apache.hc.core5.http.HttpEntity;
+import org.apache.hc.core5.http.HttpHost;
+import org.apache.hc.core5.http.HttpStatus;
+import org.apache.hc.core5.http.io.entity.ByteArrayEntity;
+import org.apache.hc.core5.http.io.entity.EntityUtils;
+import org.apache.hc.core5.net.URIBuilder;
import org.apache.hop.core.Const;
import org.apache.hop.core.encryption.Encr;
import org.apache.hop.core.exception.HopException;
@@ -71,22 +88,6 @@ import
org.apache.hop.pipeline.transforms.webservices.wsdl.WsdlOpParameter;
import org.apache.hop.pipeline.transforms.webservices.wsdl.WsdlOpParameterList;
import org.apache.hop.pipeline.transforms.webservices.wsdl.WsdlOperation;
import org.apache.hop.pipeline.transforms.webservices.wsdl.XsdType;
-import org.apache.http.Header;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpHost;
-import org.apache.http.HttpResponse;
-import org.apache.http.HttpStatus;
-import org.apache.http.client.AuthCache;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.protocol.HttpClientContext;
-import org.apache.http.client.utils.URIBuilder;
-import org.apache.http.entity.ByteArrayEntity;
-import org.apache.http.entity.ContentType;
-import org.apache.http.impl.auth.BasicScheme;
-import org.apache.http.impl.client.BasicAuthCache;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.util.EntityUtils;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
@@ -389,21 +390,25 @@ public class WebService extends
BaseTransform<WebServiceMeta, WebServiceData> {
vHttpMethod = getHttpMethod(cachedURLService);
- HttpEntity requestEntity = new
ByteArrayEntity(xml.getBytes(StandardCharsets.UTF_8));
+ HttpEntity requestEntity =
+ new ByteArrayEntity(xml.getBytes(StandardCharsets.UTF_8),
ContentType.TEXT_XML);
vHttpMethod.setEntity(requestEntity);
- HttpResponse httpResponse = cachedHttpClient.execute(vHttpMethod);
- int responseCode = httpResponse.getStatusLine().getStatusCode();
+ org.apache.hc.core5.http.ClassicHttpResponse httpResponse =
+ (org.apache.hc.core5.http.ClassicHttpResponse)
+ cachedHttpClient.execute(vHttpMethod, cachedHostConfiguration);
+ int responseCode = httpResponse.getCode();
if (responseCode == HttpStatus.SC_MOVED_PERMANENTLY) {
String newLocation = getLocationFrom(vHttpMethod);
vHttpMethod = getHttpMethod(newLocation);
vHttpMethod.setEntity(requestEntity);
- httpResponse = cachedHttpClient.execute(vHttpMethod);
- responseCode = httpResponse.getStatusLine().getStatusCode();
+ httpResponse =
+ (org.apache.hc.core5.http.ClassicHttpResponse)
+ cachedHttpClient.execute(vHttpMethod, cachedHostConfiguration);
+ responseCode = httpResponse.getCode();
}
if (responseCode == HttpStatus.SC_OK) {
httpEntity = httpResponse.getEntity();
- ContentType contentType = ContentType.getOrDefault(httpEntity);
- charSet = contentType.getCharset();
+ charSet = StandardCharsets.UTF_8;
processRows(
httpEntity.getContent(),
rowData,
@@ -424,7 +429,7 @@ public class WebService extends
BaseTransform<WebServiceMeta, WebServiceData> {
PKG,
"WebServices.ERROR0001.ServerError",
Integer.toString(responseCode),
- Const.NVL(EntityUtils.toString(httpEntity,
charSet.toString()), ""),
+ Const.NVL(readEntity(httpEntity, charSet.toString()), ""),
cachedURLService));
}
} catch (UnknownHostException e) {
@@ -438,9 +443,6 @@ public class WebService extends
BaseTransform<WebServiceMeta, WebServiceData> {
BaseMessages.getString(PKG, "WebServices.ERROR0002.InvalidURI",
cachedURLService), e);
} finally {
data.argumentRows.clear(); // ready for the next batch.
- if (vHttpMethod != null) {
- vHttpMethod.releaseConnection();
- }
}
}
@@ -465,7 +467,7 @@ public class WebService extends
BaseTransform<WebServiceMeta, WebServiceData> {
cachedURLService = cachedWsdl.getServiceEndpoint();
cachedHostConfiguration = HttpClientContext.create();
- cachedHttpClient = getHttpClient(cachedHostConfiguration);
+ cachedHttpClient = getHttpClient(cachedHostConfiguration,
cachedURLService);
// Generate the XML to send over, determine the correct name for the
request...
//
cachedOperation = cachedWsdl.getOperation(meta.getOperationName());
@@ -504,37 +506,42 @@ public class WebService extends
BaseTransform<WebServiceMeta, WebServiceData> {
return vHttpMethod;
}
- private HttpClient getHttpClient(HttpClientContext context) {
+ private HttpClient getHttpClient(HttpClientContext context, String
serviceUrl) {
HttpClientManager.HttpClientBuilderFacade clientBuilder =
HttpClientManager.getInstance().createBuilder();
String login = resolve(meta.getHttpLogin());
+ String password =
Encr.decryptPasswordOptionallyEncrypted(resolve(meta.getHttpPassword()));
if (StringUtils.isNotBlank(login)) {
- clientBuilder.setCredentials(
- login,
Encr.decryptPasswordOptionallyEncrypted(resolve(meta.getHttpPassword())));
+ clientBuilder.setCredentials(login, password);
}
- int proxyPort = 0;
if (StringUtils.isNotBlank(meta.getProxyHost())) {
- proxyPort = Const.toInt(resolve(meta.getProxyPort()), 8080);
+ int proxyPort = Const.toInt(resolve(meta.getProxyPort()), 8080);
clientBuilder.setProxy(meta.getProxyHost(), proxyPort);
}
CloseableHttpClient httpClient = clientBuilder.build();
- if (proxyPort != 0) {
- // Preemptive authentication
- HttpHost target = new HttpHost(meta.getProxyHost(), proxyPort, "http");
- // Create AuthCache instance
+ if (StringUtils.isNotBlank(login)) {
+ HttpHost target = HttpHost.create(URI.create(serviceUrl));
AuthCache authCache = new BasicAuthCache();
- // Generate BASIC scheme object and add it to the local auth cache
BasicScheme basicAuth = new BasicScheme();
+ char[] passwordChars = password != null ? password.toCharArray() : new
char[0];
+ basicAuth.initPreemptive(new UsernamePasswordCredentials(login,
passwordChars));
authCache.put(target, basicAuth);
- // Add AuthCache to the execution context
context.setAuthCache(authCache);
}
return httpClient;
}
+ private static String readEntity(HttpEntity entity, String charset) throws
IOException {
+ try {
+ return EntityUtils.toString(entity, charset);
+ } catch (org.apache.hc.core5.http.ParseException e) {
+ throw new IOException("Unable to parse SOAP response body", e);
+ }
+ }
+
@Override
public boolean init() {
diff --git
a/plugins/transforms/webservices/src/main/java/org/apache/hop/pipeline/transforms/webservices/wsdl/Wsdl.java
b/plugins/transforms/webservices/src/main/java/org/apache/hop/pipeline/transforms/webservices/wsdl/Wsdl.java
index 6125e89175..ecbaeb198c 100644
---
a/plugins/transforms/webservices/src/main/java/org/apache/hop/pipeline/transforms/webservices/wsdl/Wsdl.java
+++
b/plugins/transforms/webservices/src/main/java/org/apache/hop/pipeline/transforms/webservices/wsdl/Wsdl.java
@@ -18,10 +18,15 @@
package org.apache.hop.pipeline.transforms.webservices.wsdl;
import java.io.IOException;
+import java.io.InputStream;
import java.io.Serial;
import java.net.MalformedURLException;
import java.net.URI;
+import java.net.URL;
+import java.net.URLConnection;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
+import java.util.Base64;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
@@ -38,12 +43,10 @@ import javax.wsdl.factory.WSDLFactory;
import javax.wsdl.xml.WSDLLocator;
import javax.wsdl.xml.WSDLReader;
import javax.xml.namespace.QName;
-import org.apache.hop.core.HttpProtocol;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.exception.HopTransformException;
import org.apache.hop.core.logging.LogChannel;
import org.apache.hop.core.xml.XmlHandler;
-import org.apache.http.auth.AuthenticationException;
import org.w3c.dom.Document;
/** Wsdl abstraction. */
@@ -64,19 +67,15 @@ public final class Wsdl implements java.io.Serializable {
* @param serviceQName Name of the service in the WSDL, if null default to
first service in WSDL.
* @param portName The service port name, if null default to first port in
service.
*/
- public Wsdl(URI wsdlURI, QName serviceQName, String portName) throws
AuthenticationException {
+ public Wsdl(URI wsdlURI, QName serviceQName, String portName) {
this(wsdlURI, serviceQName, portName, null, null);
}
- public Wsdl(URI wsdlURI, QName serviceQName, String portName, String
username, String password)
- throws AuthenticationException {
+ public Wsdl(URI wsdlURI, QName serviceQName, String portName, String
username, String password) {
this.wsdlURI = wsdlURI;
try {
wsdlDefinition = parse(wsdlURI, username, password);
- } catch (AuthenticationException ae) {
- // throw this again since HopException is catching it
- throw ae;
} catch (WSDLException | HopException e) {
throw new RuntimeException(CONST_COULD_NOT_LOAD_WSDL_FILE +
e.getMessage(), e);
}
@@ -132,8 +131,7 @@ public final class Wsdl implements java.io.Serializable {
* @param serviceQName Name of the service in the WSDL.
* @param portName The service port name.
*/
- public Wsdl(WSDLLocator wsdlLocator, QName serviceQName, String portName)
- throws AuthenticationException {
+ public Wsdl(WSDLLocator wsdlLocator, QName serviceQName, String portName) {
this(wsdlLocator, serviceQName, portName, null, null);
}
@@ -142,15 +140,11 @@ public final class Wsdl implements java.io.Serializable {
QName serviceQName,
String portName,
String username,
- String password)
- throws AuthenticationException {
+ String password) {
// load and parse the WSDL
try {
wsdlDefinition = parse(wsdlLocator, username, password);
- } catch (AuthenticationException ae) {
- // throw it again or HopException will catch it
- throw ae;
} catch (WSDLException | HopException e) {
throw new RuntimeException(CONST_COULD_NOT_LOAD_WSDL_FILE +
e.getMessage(), e);
}
@@ -326,7 +320,7 @@ public final class Wsdl implements java.io.Serializable {
* @throws WSDLException on error.
*/
private Definition parse(WSDLLocator wsdlLocator, String username, String
password)
- throws WSDLException, HopException, AuthenticationException {
+ throws WSDLException, HopException {
WSDLReader wsdlReader = getReader();
try {
@@ -348,33 +342,39 @@ public final class Wsdl implements java.io.Serializable {
* @throws WSDLException on error.
*/
private Definition parse(URI wsdlURI, String username, String password)
- throws WSDLException, HopException, AuthenticationException {
+ throws WSDLException, HopException {
WSDLReader wsdlReader = getReader();
return readWsdl(wsdlReader, wsdlURI.toString(), username, password);
}
private Definition readWsdl(WSDLReader wsdlReader, String uri, String
username, String password)
- throws WSDLException, HopException, AuthenticationException {
+ throws WSDLException, HopException {
- try {
- HttpProtocol http = new HttpProtocol();
- Document doc =
- XmlHandler.loadXmlString(http.get(wsdlURI.toString(), username,
password), true, false);
+ try (InputStream wsdlStream = openWsdlStream(uri, username, password)) {
+ Document doc = XmlHandler.loadXmlFile(wsdlStream, uri, false, true);
if (doc != null) {
- return (wsdlReader.readWSDL(doc.getBaseURI(), doc));
+ return wsdlReader.readWSDL(uri, doc);
} else {
throw new HopException("Unable to get document.");
}
} catch (MalformedURLException mue) {
throw new HopException(mue);
- } catch (AuthenticationException ae) {
- // re-throw this. If not IOException seems to catch it
- throw ae;
} catch (IOException ioe) {
throw new HopException(ioe);
}
}
+ private InputStream openWsdlStream(String uri, String username, String
password)
+ throws IOException {
+ URLConnection connection = new URL(uri).openConnection();
+ if (username != null && !username.isEmpty()) {
+ String raw = username + ":" + (password == null ? "" : password);
+ String encoded =
Base64.getEncoder().encodeToString(raw.getBytes(StandardCharsets.UTF_8));
+ connection.setRequestProperty("Authorization", "Basic " + encoded);
+ }
+ return connection.getInputStream();
+ }
+
/**
* Returns this objects WSDL types.
*
diff --git
a/plugins/transforms/webservices/src/test/java/org/apache/hop/pipeline/transforms/webservices/WebServiceTest.java
b/plugins/transforms/webservices/src/test/java/org/apache/hop/pipeline/transforms/webservices/WebServiceTest.java
index 4d9e4131f0..61a58051bf 100644
---
a/plugins/transforms/webservices/src/test/java/org/apache/hop/pipeline/transforms/webservices/WebServiceTest.java
+++
b/plugins/transforms/webservices/src/test/java/org/apache/hop/pipeline/transforms/webservices/WebServiceTest.java
@@ -26,11 +26,11 @@ import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
import java.net.URISyntaxException;
+import org.apache.hc.client5.http.classic.methods.HttpPost;
+import org.apache.hc.core5.http.Header;
+import org.apache.hc.core5.http.message.BasicHeader;
import org.apache.hop.core.logging.ILoggingObject;
import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
-import org.apache.http.Header;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.message.BasicHeader;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
diff --git
a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/getxmldata/GetXmlData.java
b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/getxmldata/GetXmlData.java
index 92266289f1..328297662f 100644
---
a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/getxmldata/GetXmlData.java
+++
b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/getxmldata/GetXmlData.java
@@ -28,6 +28,11 @@ import java.util.List;
import java.util.zip.GZIPInputStream;
import org.apache.commons.vfs2.FileObject;
import org.apache.commons.vfs2.FileSystemException;
+import org.apache.hc.client5.http.classic.HttpClient;
+import org.apache.hc.client5.http.classic.methods.HttpGet;
+import org.apache.hc.core5.http.ClassicHttpResponse;
+import org.apache.hc.core5.http.Header;
+import org.apache.hc.core5.http.HttpEntity;
import org.apache.hop.core.Const;
import org.apache.hop.core.ResultFile;
import org.apache.hop.core.exception.HopException;
@@ -47,11 +52,6 @@ import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.BaseTransform;
import org.apache.hop.pipeline.transform.TransformMeta;
import org.apache.hop.pipeline.transforms.xml.Dom4JUtil;
-import org.apache.http.Header;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.HttpGet;
import org.dom4j.Element;
import org.dom4j.ElementHandler;
import org.dom4j.ElementPath;
@@ -189,7 +189,7 @@ public class GetXmlData extends
BaseTransform<GetXmlDataMeta, GetXmlDataData> {
HttpClient client =
HttpClientManager.getInstance().createDefaultClient();
HttpGet method = new HttpGet(stringXML);
method.addHeader("Accept-Encoding", "gzip");
- HttpResponse response = client.execute(method);
+ ClassicHttpResponse response = (ClassicHttpResponse)
client.execute(method);
Header contentEncoding = response.getFirstHeader("Content-Encoding");
HttpEntity responseEntity = response.getEntity();
if (responseEntity != null) {
diff --git
a/ui/src/main/java/org/apache/hop/ui/www/service/WebServiceEditor.java
b/ui/src/main/java/org/apache/hop/ui/www/service/WebServiceEditor.java
index c0c8383be7..6f1ec6654b 100644
--- a/ui/src/main/java/org/apache/hop/ui/www/service/WebServiceEditor.java
+++ b/ui/src/main/java/org/apache/hop/ui/www/service/WebServiceEditor.java
@@ -19,6 +19,7 @@
package org.apache.hop.ui.www.service;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hc.core5.http.ContentType;
import org.apache.hop.core.Const;
import org.apache.hop.core.NotePadMeta;
import org.apache.hop.core.logging.LogChannel;
@@ -40,7 +41,6 @@ import org.apache.hop.ui.core.widget.TextVar;
import org.apache.hop.ui.hopgui.HopGui;
import org.apache.hop.ui.hopgui.file.pipeline.HopPipelineFileType;
import org.apache.hop.www.service.WebService;
-import org.apache.http.entity.ContentType;
import org.eclipse.swt.SWT;
import org.eclipse.swt.layout.FormAttachment;
import org.eclipse.swt.layout.FormData;
