This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/dev/pr-13011-a3e1a731a3ca6d7136e26d9d46ca685d9675c4da in repository https://gitbox.apache.org/repos/asf/seatunnel.git
commit 9cd7defe84e27c725ecf6dcf588be94f63477d4a Author: corgy-w <[email protected]> AuthorDate: Wed Nov 11 14:39:14 2026 +0000 [Fix][Connector-V2] Encode the HTTP source response body as UTF-8 (#13011) --- .../seatunnel/http/source/HttpSourceReader.java | 3 +- .../http/HttpSourceReaderInternalPollNextTest.java | 34 ++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/source/HttpSourceReader.java b/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/source/HttpSourceReader.java index 7efd9c6a24..b869f80a51 100644 --- a/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/source/HttpSourceReader.java +++ b/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/source/HttpSourceReader.java @@ -51,6 +51,7 @@ import lombok.extern.slf4j.Slf4j; import java.io.BufferedReader; import java.io.IOException; import java.io.StringReader; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -498,7 +499,7 @@ public class HttpSourceReader extends AbstractSingleSplitReader<SeaTunnelRow> { } } } - deserializationCollector.collect(contentData.getBytes(), output); + deserializationCollector.collect(contentData.getBytes(StandardCharsets.UTF_8), output); } private List<List<String>> decodeJSON(String data) { diff --git a/seatunnel-connectors-v2/connector-http/connector-http-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/http/HttpSourceReaderInternalPollNextTest.java b/seatunnel-connectors-v2/connector-http/connector-http-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/http/HttpSourceReaderInternalPollNextTest.java index 5e4bda1ab1..e189859388 100644 --- a/seatunnel-connectors-v2/connector-http/connector-http-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/http/HttpSourceReaderInternalPollNextTest.java +++ b/seatunnel-connectors-v2/connector-http/connector-http-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/http/HttpSourceReaderInternalPollNextTest.java @@ -16,6 +16,7 @@ */ package org.apache.seatunnel.connectors.seatunnel.http; +import org.apache.seatunnel.api.serialization.DeserializationSchema; import org.apache.seatunnel.api.source.Boundedness; import org.apache.seatunnel.api.source.Collector; import org.apache.seatunnel.api.table.type.BasicType; @@ -38,9 +39,12 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.MockitoAnnotations; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; import java.util.concurrent.CountDownLatch; @@ -247,4 +251,34 @@ public class HttpSourceReaderInternalPollNextTest { public void tearDown() throws Exception { mock.close(); } + + /** + * The response body reaches the format deserializer as bytes, and those bytes must always be + * UTF-8: the format deserializers decode with UTF-8, so encoding the body with the JVM default + * charset corrupts non-ASCII values whenever that default is not UTF-8 (the Windows CI legs run + * JDK 8/11, whose default charset is Cp1252). + */ + @Test + public void testResponseBodyIsCollectedAsUtf8RegardlessOfDefaultCharset() throws Exception { + String body = "[{\"key1\":\"\u4e2d\u6587\u503c\",\"key2\":\"v2\"}]"; + when(httpClientProvider.execute( + anyString(), anyString(), any(), any(), any(), anyBoolean())) + .thenReturn(new HttpResponse(200, body)); + + DeserializationSchema<SeaTunnelRow> schema = Mockito.mock(DeserializationSchema.class); + HttpSourceReader reader = + new HttpSourceReader(httpParameter, context, schema, null, null, null); + reader.open(); + reader.setHttpClient(httpClientProvider); + + reader.internalPollNext(collector); + + ArgumentCaptor<byte[]> message = ArgumentCaptor.forClass(byte[].class); + verify(schema).deserialize(message.capture()); + Assertions.assertEquals( + body, + new String(message.getValue(), StandardCharsets.UTF_8), + "the response body must reach the deserializer as UTF-8 bytes"); + reader.close(); + } }
