This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 49180f65fc Issue #8507 : Do not send a charset on the REST Client
Content-Type (#8569)
49180f65fc is described below
commit 49180f65fcb52cf2a9c3e0636ff67c06e544f36c
Author: Matt Casters <[email protected]>
AuthorDate: Fri Sep 25 08:10:40 2026 +0200
Issue #8507 : Do not send a charset on the REST Client Content-Type (#8569)
* Issue #8507 : Do not send a charset on the REST Client Content-Type
HttpClient's JSON content type is application/json; charset=UTF-8. Omie and
similar gateways answer that parameter with HTTP 500 and a SOAP Sender
fault.
Send the mime type alone unless the row sets Content-Type itself. The body
encoding is unchanged.
* Address review feedback: omit charset on Content-Type for JSON only
(#8507)
---
.../apache/hop/pipeline/transforms/rest/Rest.java | 25 ++--
.../pipeline/transforms/rest/RestCallRestTest.java | 142 +++++++++++++++++++++
.../transforms/rest/RestInitAndProcessTest.java | 7 +-
.../transforms/rest/RestRequestLoggingTest.java | 8 +-
4 files changed, 166 insertions(+), 16 deletions(-)
diff --git
a/plugins/transforms/rest/src/main/java/org/apache/hop/pipeline/transforms/rest/Rest.java
b/plugins/transforms/rest/src/main/java/org/apache/hop/pipeline/transforms/rest/Rest.java
index 08df19ab32..ab51965bfc 100644
---
a/plugins/transforms/rest/src/main/java/org/apache/hop/pipeline/transforms/rest/Rest.java
+++
b/plugins/transforms/rest/src/main/java/org/apache/hop/pipeline/transforms/rest/Rest.java
@@ -68,7 +68,6 @@ import org.apache.hc.core5.http.Header;
import org.apache.hc.core5.http.HttpEntity;
import org.apache.hc.core5.http.io.entity.ByteArrayEntity;
import org.apache.hc.core5.http.io.entity.EntityUtils;
-import org.apache.hc.core5.http.io.entity.StringEntity;
import org.apache.hc.core5.net.URIBuilder;
import org.apache.hop.core.Const;
import org.apache.hop.core.encryption.Encr;
@@ -1644,11 +1643,10 @@ public class Rest extends BaseTransform<RestMeta,
RestData> {
if (RestMeta.isActiveBody(data.method)) {
ContentType type = contentType != null ?
ContentType.parse(contentType) : data.mediaType;
- trackRequestBytes(body, resolveCharset(type));
- request.setEntity(
- body instanceof byte[] bytes
- ? new ByteArrayEntity(bytes, type)
- : new StringEntity((String) body, type));
+ Charset charset = resolveCharset(type);
+ trackRequestBytes(body, charset);
+ byte[] payload = body instanceof byte[] bytes ? bytes : ((String)
body).getBytes(charset);
+ request.setEntity(new ByteArrayEntity(payload, type));
}
if (isDetailed()) {
@@ -1701,14 +1699,16 @@ public class Rest extends BaseTransform<RestMeta,
RestData> {
.append(CredentialRedactor.redact(request.getRequestUri()))
.append(Const.CR);
- // Host and Content-Type never appear in getHeaders(): the client derives
the first from the
- // route and the second from the entity, both at send time. Leaving them
out would make this a
- // misleading picture of the request rather than a faithful one.
+ // Host is filled in from the route at send time, so it is not in
getHeaders() yet.
+ // Content-Type is already on the request when the row set it. Otherwise
it lives on
+ // the entity and is copied at send time. Print the one that will go out,
once.
if (request.getAuthority() != null) {
text.append("Host:
").append(request.getAuthority().toString()).append(Const.CR);
}
HttpEntity requestEntity = request.getEntity();
- if (requestEntity != null && requestEntity.getContentType() != null) {
+ if (!request.containsHeader("Content-Type")
+ && requestEntity != null
+ && requestEntity.getContentType() != null) {
text.append("Content-Type:
").append(requestEntity.getContentType()).append(Const.CR);
}
@@ -2264,7 +2264,10 @@ public class Rest extends BaseTransform<RestMeta,
RestData> {
String applicationType = NVL(meta.getApplicationType(), "");
switch (applicationType) {
case RestMeta.APPLICATION_TYPE_XML -> data.mediaType =
ContentType.APPLICATION_XML;
- case RestMeta.APPLICATION_TYPE_JSON -> data.mediaType =
ContentType.APPLICATION_JSON;
+ // Issue #8507: ContentType.APPLICATION_JSON has charset=UTF-8,
which gateways such as
+ // Omie reject. JSON is defined as UTF-8 (RFC 8259), so omit the
charset parameter.
+ case RestMeta.APPLICATION_TYPE_JSON ->
+ data.mediaType = ContentType.create("application/json");
case RestMeta.APPLICATION_TYPE_OCTET_STREAM ->
data.mediaType = ContentType.APPLICATION_OCTET_STREAM;
case RestMeta.APPLICATION_TYPE_XHTML -> data.mediaType =
ContentType.APPLICATION_XHTML_XML;
diff --git
a/plugins/transforms/rest/src/test/java/org/apache/hop/pipeline/transforms/rest/RestCallRestTest.java
b/plugins/transforms/rest/src/test/java/org/apache/hop/pipeline/transforms/rest/RestCallRestTest.java
index a53d4fa10e..0958f41b6d 100644
---
a/plugins/transforms/rest/src/test/java/org/apache/hop/pipeline/transforms/rest/RestCallRestTest.java
+++
b/plugins/transforms/rest/src/test/java/org/apache/hop/pipeline/transforms/rest/RestCallRestTest.java
@@ -17,8 +17,11 @@
package org.apache.hop.pipeline.transforms.rest;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
@@ -214,6 +217,109 @@ class RestCallRestTest {
assertEquals("", requestBody());
}
+ /**
+ * Issue #8507. {@link ContentType#APPLICATION_JSON} is {@code
application/json; charset=UTF-8}.
+ * Sending that parameter makes gateways such as Omie answer HTTP 500 with a
SOAP Sender fault
+ * before the API runs. The body stays UTF-8. The charset is omitted unless
the row sets
+ * Content-Type itself.
+ */
+ @Test
+ void testJsonPostOmitsCharsetParameter() throws Exception {
+ Rest rest = post();
+ rest.callRest(new Object[] {"value1", "{\"n\":\"caf\u00e9\"}"});
+
+ ClassicHttpRequest request = FakeHttpClient.captured();
+ assertNull(request.getFirstHeader("Content-Type"));
+ assertEquals("application/json", request.getEntity().getContentType());
+
assertArrayEquals("{\"n\":\"caf\u00e9\"}".getBytes(StandardCharsets.UTF_8),
requestBytes());
+ }
+
+ @Test
+ void testExplicitContentTypeWithoutCharsetIsKept() throws Exception {
+ Rest rest = post("Content-Type", "application/json");
+ rest.callRest(new Object[] {"value1", "{\"n\":\"caf\u00e9\"}",
"application/json"});
+
+ ClassicHttpRequest request = FakeHttpClient.captured();
+ assertEquals(1, request.getHeaders("Content-Type").length);
+ assertEquals("application/json",
request.getFirstHeader("Content-Type").getValue());
+
assertArrayEquals("{\"n\":\"caf\u00e9\"}".getBytes(StandardCharsets.UTF_8),
requestBytes());
+ }
+
+ @Test
+ void testExplicitContentTypeCharsetIsHonored() throws Exception {
+ String header = "application/json; charset=ISO-8859-1";
+ Rest rest = post("Content-Type", header);
+ rest.callRest(new Object[] {"value1", "caf\u00e9", header});
+
+ ClassicHttpRequest request = FakeHttpClient.captured();
+ assertEquals(1, request.getHeaders("Content-Type").length);
+ assertEquals(header, request.getFirstHeader("Content-Type").getValue());
+ assertArrayEquals("caf\u00e9".getBytes(StandardCharsets.ISO_8859_1),
requestBytes());
+ }
+
+ @Test
+ void testFormUrlEncodedPostRetainsCharsetParameter() throws Exception {
+ RestMeta meta = new RestMeta();
+ meta.setMethod(RestMeta.HTTP_METHOD_POST);
+ meta.setUrl("http://example.com/api";);
+ meta.setBodyField("body");
+ meta.setResultField(new ResultField());
+ meta.getResultField().setFieldName("result");
+
+ RestData data = new RestData();
+ data.mediaType = ContentType.APPLICATION_FORM_URLENCODED;
+ data.method = RestMeta.HTTP_METHOD_POST;
+ data.realUrl = "http://example.com/api";;
+ data.resultFieldName = "result";
+ data.useBody = true;
+ data.indexOfBodyField = 1;
+ data.inputRowMeta = rowMeta("field1", "body");
+
+ Rest rest = transform(meta, data, json(200, "{}"));
+ rest.callRest(new Object[] {"value1", "a=caf\u00e9"});
+
+ ClassicHttpRequest request = FakeHttpClient.captured();
+ assertNull(request.getFirstHeader("Content-Type"));
+ assertEquals(
+ ContentType.APPLICATION_FORM_URLENCODED.toString(),
request.getEntity().getContentType());
+ assertArrayEquals("a=caf\u00e9".getBytes(StandardCharsets.ISO_8859_1),
requestBytes());
+ }
+
+ @Test
+ void testTextPlainPostRetainsCharsetParameter() throws Exception {
+ RestMeta meta = new RestMeta();
+ meta.setMethod(RestMeta.HTTP_METHOD_POST);
+ meta.setUrl("http://example.com/api";);
+ meta.setBodyField("body");
+ meta.setResultField(new ResultField());
+ meta.getResultField().setFieldName("result");
+
+ RestData data = new RestData();
+ data.mediaType = ContentType.TEXT_PLAIN;
+ data.method = RestMeta.HTTP_METHOD_POST;
+ data.realUrl = "http://example.com/api";;
+ data.resultFieldName = "result";
+ data.useBody = true;
+ data.indexOfBodyField = 1;
+ data.inputRowMeta = rowMeta("field1", "body");
+
+ Rest rest = transform(meta, data, json(200, "{}"));
+ rest.callRest(new Object[] {"value1", "caf\u00e9"});
+
+ ClassicHttpRequest request = FakeHttpClient.captured();
+ assertNull(request.getFirstHeader("Content-Type"));
+ assertEquals(ContentType.TEXT_PLAIN.toString(),
request.getEntity().getContentType());
+ assertArrayEquals("caf\u00e9".getBytes(StandardCharsets.UTF_8),
requestBytes());
+ }
+
+ @Test
+ void testMalformedRowContentTypeFails() {
+ Rest rest = post("Content-Type", "application/json; charset=utf-99");
+ assertThrows(
+ Exception.class,
+ () -> rest.callRest(new Object[] {"value1", "{}", "application/json;
charset=utf-99"}));
+ }
+
@Test
void testCallRestWithPutMethod() throws HopException {
RestMeta meta = new RestMeta();
@@ -576,6 +682,34 @@ class RestCallRestTest {
return rowMeta;
}
+ /** A JSON POST. When two arguments are given they are a header name and the
row value for it. */
+ private static Rest post(String... header) {
+ RestMeta meta = new RestMeta();
+ meta.setMethod(RestMeta.HTTP_METHOD_POST);
+ meta.setUrl("http://example.com/api";);
+ meta.setBodyField("body");
+ meta.setResultField(new ResultField());
+ meta.getResultField().setFieldName("result");
+
+ RestData data = new RestData();
+ data.mediaType = ContentType.create("application/json");
+ data.method = RestMeta.HTTP_METHOD_POST;
+ data.realUrl = "http://example.com/api";;
+ data.resultFieldName = "result";
+ data.useBody = true;
+ data.indexOfBodyField = 1;
+ if (header.length == 2) {
+ data.useHeaders = true;
+ data.nrheader = 1;
+ data.headerNames = new String[] {header[0]};
+ data.indexOfHeaderFields = new int[] {2};
+ data.inputRowMeta = rowMeta("field1", "body", "header");
+ } else {
+ data.inputRowMeta = rowMeta("field1", "body");
+ }
+ return transform(meta, data, json(200, "{}"));
+ }
+
/** The URL the captured request was actually sent to. */
private static String uri() {
try {
@@ -592,4 +726,12 @@ class RestCallRestTest {
throw new IllegalStateException("Unable to read the captured request
body", e);
}
}
+
+ private static byte[] requestBytes() {
+ try {
+ return EntityUtils.toByteArray(FakeHttpClient.captured().getEntity());
+ } catch (Exception e) {
+ throw new IllegalStateException("Unable to read the captured request
body", e);
+ }
+ }
}
diff --git
a/plugins/transforms/rest/src/test/java/org/apache/hop/pipeline/transforms/rest/RestInitAndProcessTest.java
b/plugins/transforms/rest/src/test/java/org/apache/hop/pipeline/transforms/rest/RestInitAndProcessTest.java
index 49e710f249..5be672584c 100644
---
a/plugins/transforms/rest/src/test/java/org/apache/hop/pipeline/transforms/rest/RestInitAndProcessTest.java
+++
b/plugins/transforms/rest/src/test/java/org/apache/hop/pipeline/transforms/rest/RestInitAndProcessTest.java
@@ -20,6 +20,7 @@ package org.apache.hop.pipeline.transforms.rest;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
@@ -102,7 +103,8 @@ class RestInitAndProcessTest {
assertEquals(5000, data.realConnectionTimeout);
assertEquals(10000, data.realReadTimeout);
assertEquals(RestMeta.HTTP_METHOD_GET, data.method);
- assertEquals(ContentType.APPLICATION_JSON, data.mediaType);
+ assertEquals("application/json", data.mediaType.toString());
+ assertNull(data.mediaType.getCharset());
}
@Test
@@ -230,7 +232,8 @@ class RestInitAndProcessTest {
assertEquals(ContentType.APPLICATION_XML, data.mediaType);
break;
case RestMeta.APPLICATION_TYPE_JSON:
- assertEquals(ContentType.APPLICATION_JSON, data.mediaType);
+ assertEquals("application/json", data.mediaType.toString());
+ assertNull(data.mediaType.getCharset());
break;
case RestMeta.APPLICATION_TYPE_OCTET_STREAM:
assertEquals(ContentType.APPLICATION_OCTET_STREAM, data.mediaType);
diff --git
a/plugins/transforms/rest/src/test/java/org/apache/hop/pipeline/transforms/rest/RestRequestLoggingTest.java
b/plugins/transforms/rest/src/test/java/org/apache/hop/pipeline/transforms/rest/RestRequestLoggingTest.java
index 26f198a383..bd038fb421 100644
---
a/plugins/transforms/rest/src/test/java/org/apache/hop/pipeline/transforms/rest/RestRequestLoggingTest.java
+++
b/plugins/transforms/rest/src/test/java/org/apache/hop/pipeline/transforms/rest/RestRequestLoggingTest.java
@@ -64,10 +64,12 @@ class RestRequestLoggingTest {
assertTrue(
log.contains("/api/things?q=hop"), "the request line carries the query
string:\n" + log);
assertTrue(log.contains("X-Trace: abc123"), "a configured header:\n" +
log);
- // Neither of these is a header on the request object: the client derives
Host from the route
- // and Content-Type from the entity. Without them the block would
misrepresent the request.
+ // Host is not on the request object yet: the client derives it from the
route at send time.
+ // Content-Type is on the entity until then, and the log prints the value
that will be sent.
+ // Issue #8507: that value is the mime type, without a charset parameter.
assertTrue(log.contains("Host: example.com"), "the host:\n" + log);
assertTrue(log.contains("Content-Type: application/json"), "the content
type:\n" + log);
+ assertFalse(log.toLowerCase().contains("charset"), "no charset
parameter:\n" + log);
assertTrue(log.contains("{\"name\":\"hop\"}"), "the body:\n" + log);
}
@@ -138,7 +140,7 @@ class RestRequestLoggingTest {
meta.getResultField().setFieldName("result");
RestData data = new RestData();
- data.mediaType = ContentType.APPLICATION_JSON;
+ data.mediaType = ContentType.create("application/json");
data.method = method;
data.realUrl = "http://example.com/api/things";;
data.resultFieldName = "result";
