This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/dev/pr-13139-d208f66bffa6452da7bc229af4970b9616850cd4 in repository https://gitbox.apache.org/repos/asf/seatunnel.git
commit f29d56b05d0dd506075e060a0f8fa0e5950534a2 Author: JacobZheng0927 <[email protected]> AuthorDate: Tue Nov 17 15:55:24 2026 +0000 [Fix][Connector-V2] Normalize Arrow boolean values for TINYINT schemas (#13139) --- docs/en/connectors/source/StarRocks.md | 4 ++ docs/zh/connectors/source/StarRocks.md | 3 ++ .../arrow/reader/ArrowToSeatunnelRowReader.java | 5 +++ .../arrow/ArrowToSeatunnelRowReaderTest.java | 48 ++++++++++++++++++++++ .../connector-starrocks-e2e/pom.xml | 6 +++ .../e2e/connector/starrocks/StarRocksIT.java | 47 +++++++++++++++++++++ .../starrocks-boolean-to-jdbc-tinyint.conf | 48 ++++++++++++++++++++++ 7 files changed, 161 insertions(+) diff --git a/docs/en/connectors/source/StarRocks.md b/docs/en/connectors/source/StarRocks.md index f46d9e5da0..7a29193ba1 100644 --- a/docs/en/connectors/source/StarRocks.md +++ b/docs/en/connectors/source/StarRocks.md @@ -78,6 +78,10 @@ e.g. The schema of the StarRocks rows that SeaTunnel will emit. For more details, please refer to [Schema Feature](../../introduction/concepts/schema-feature.md). +When a boolean value returned by StarRocks is declared as `TINYINT` in the source schema, +SeaTunnel converts `false` to `0` and `true` to `1` as byte values, preserving `null`. +Declaring the field as `BOOLEAN` preserves boolean values. + e.g. ``` diff --git a/docs/zh/connectors/source/StarRocks.md b/docs/zh/connectors/source/StarRocks.md index cb0eb54872..7aa164081b 100644 --- a/docs/zh/connectors/source/StarRocks.md +++ b/docs/zh/connectors/source/StarRocks.md @@ -78,6 +78,9 @@ import ChangeLog from '../changelog/connector-starrocks.md'; SeaTunnel 输出的 StarRocks 数据结构。更多详情请参考 [Schema 特性](../../introduction/concepts/schema-feature.md)。 +当 StarRocks 返回布尔值,而源 schema 将该字段声明为 `TINYINT` 时,SeaTunnel 会将 `false` 转为字节值 `0`, +将 `true` 转为字节值 `1`,并保留 `null`。声明为 `BOOLEAN` 时保持布尔值。 + 示例 ``` diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/reader/ArrowToSeatunnelRowReader.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/reader/ArrowToSeatunnelRowReader.java index 621e70cab4..56ba0a2ff6 100644 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/reader/ArrowToSeatunnelRowReader.java +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/reader/ArrowToSeatunnelRowReader.java @@ -192,6 +192,11 @@ public class ArrowToSeatunnelRowReader implements AutoCloseable { private Object convertSeatunnelRowValue( SqlType currentType, Types.MinorType minorType, Object fieldValue) { switch (currentType) { + case TINYINT: + if (fieldValue instanceof Boolean) { + return (byte) ((Boolean) fieldValue ? 1 : 0); + } + return fieldValue; case STRING: if (fieldValue instanceof byte[]) { return new String((byte[]) fieldValue); diff --git a/seatunnel-connectors-v2/connector-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/ArrowToSeatunnelRowReaderTest.java b/seatunnel-connectors-v2/connector-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/ArrowToSeatunnelRowReaderTest.java index 3e26e34cfa..fb7ae6df1d 100644 --- a/seatunnel-connectors-v2/connector-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/ArrowToSeatunnelRowReaderTest.java +++ b/seatunnel-connectors-v2/connector-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/ArrowToSeatunnelRowReaderTest.java @@ -499,6 +499,54 @@ public class ArrowToSeatunnelRowReaderTest { } } + @Test + public void testBooleanValuesFollowDeclaredSchema() throws Exception { + SeaTunnelRowType rowType = + new SeaTunnelRowType( + new String[] {"as_tinyint", "as_boolean", "numeric_tinyint"}, + new SeaTunnelDataType<?>[] { + BasicType.BYTE_TYPE, BasicType.BOOLEAN_TYPE, BasicType.BYTE_TYPE + }); + try (RootAllocator allocator = new RootAllocator(Integer.MAX_VALUE); + VectorSchemaRoot vectorRoot = + new VectorSchemaRoot( + Arrays.asList( + new BitVector("as_tinyint", allocator), + new BitVector("as_boolean", allocator), + new TinyIntVector("numeric_tinyint", allocator))); + ByteArrayOutputStream out = new ByteArrayOutputStream()) { + vectorRoot.allocateNew(); + for (int column = 0; column < 2; column++) { + BitVector vector = (BitVector) vectorRoot.getVector(column); + vector.setSafe(0, 0); + vector.setSafe(1, 1); + vector.setNull(2); + vector.setValueCount(3); + } + TinyIntVector numericVector = (TinyIntVector) vectorRoot.getVector(2); + numericVector.setSafe(0, Byte.MIN_VALUE); + numericVector.setSafe(1, Byte.MAX_VALUE); + numericVector.setNull(2); + numericVector.setValueCount(3); + vectorRoot.setRowCount(3); + try (ArrowStreamWriter writer = new ArrowStreamWriter(vectorRoot, null, out)) { + writer.start(); + writer.writeBatch(); + writer.end(); + } + try (ArrowToSeatunnelRowReader reader = + new ArrowToSeatunnelRowReader(out.toByteArray(), rowType).readArrow()) { + Assertions.assertArrayEquals( + new Object[] {(byte) 0, false, Byte.MIN_VALUE}, reader.next().getFields()); + Assertions.assertArrayEquals( + new Object[] {(byte) 1, true, Byte.MAX_VALUE}, reader.next().getFields()); + Assertions.assertArrayEquals( + new Object[] {null, null, null}, reader.next().getFields()); + Assertions.assertFalse(reader.hasNext()); + } + } + } + @Test public void testConvertArrowSpeed() throws Exception { Stopwatch stopwatch = Stopwatch.createStarted(); diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/pom.xml index 7f11b08d8b..123b80f1c2 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/pom.xml @@ -30,6 +30,12 @@ </properties> <dependencies> <!-- SeaTunnel connectors --> + <dependency> + <groupId>org.apache.seatunnel</groupId> + <artifactId>connector-jdbc</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> <dependency> <groupId>org.apache.seatunnel</groupId> <artifactId>connector-starrocks</artifactId> diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/src/test/java/org/apache/seatunnel/e2e/connector/starrocks/StarRocksIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/src/test/java/org/apache/seatunnel/e2e/connector/starrocks/StarRocksIT.java index 56b323c7ce..535014a88f 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/src/test/java/org/apache/seatunnel/e2e/connector/starrocks/StarRocksIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/src/test/java/org/apache/seatunnel/e2e/connector/starrocks/StarRocksIT.java @@ -524,6 +524,53 @@ public class StarRocksIT extends TestSuiteBase implements TestResource { Assertions.assertEquals(0, execResult.getExitCode()); } + @TestTemplate + public void testStarRocksBooleanAsTinyInt(TestContainer container) + throws IOException, InterruptedException, SQLException { + String sourceTable = DATABASE + ".e2e_boolean_as_tinyint_source"; + String sinkTable = DATABASE + ".e2e_boolean_as_tinyint_sink"; + try (Statement statement = jdbcConnection.createStatement()) { + try { + statement.execute( + "CREATE TABLE " + + sourceTable + + " (id INT, flag BOOLEAN) ENGINE=OLAP DUPLICATE KEY(id) " + + "DISTRIBUTED BY HASH(id) BUCKETS 1 " + + "PROPERTIES (\"replication_num\" = \"1\")"); + statement.execute( + "CREATE TABLE " + + sinkTable + + " (id INT, flag TINYINT) ENGINE=OLAP DUPLICATE KEY(id) " + + "DISTRIBUTED BY HASH(id) BUCKETS 1 " + + "PROPERTIES (\"replication_num\" = \"1\")"); + statement.execute( + "INSERT INTO " + sourceTable + " VALUES (1, false), (2, true), (3, null)"); + + Container.ExecResult execResult = + container.executeJob("/starrocks-boolean-to-jdbc-tinyint.conf"); + Assertions.assertEquals(0, execResult.getExitCode(), execResult.getStderr()); + try (ResultSet result = + statement.executeQuery( + "SELECT id, flag FROM " + sinkTable + " ORDER BY id")) { + Byte[] expectedValues = {(byte) 0, (byte) 1, null}; + for (int i = 0; i < expectedValues.length; i++) { + Assertions.assertTrue(result.next()); + Assertions.assertEquals(i + 1, result.getInt(1)); + Byte actual = result.getByte(2); + if (result.wasNull()) { + actual = null; + } + Assertions.assertEquals(expectedValues[i], actual); + } + Assertions.assertFalse(result.next()); + } + } finally { + statement.execute("DROP TABLE IF EXISTS " + sourceTable); + statement.execute("DROP TABLE IF EXISTS " + sinkTable); + } + } + } + @TestTemplate public void testStarRocksMultipleRead(TestContainer container) throws IOException, InterruptedException { diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/src/test/resources/starrocks-boolean-to-jdbc-tinyint.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/src/test/resources/starrocks-boolean-to-jdbc-tinyint.conf new file mode 100644 index 0000000000..ba7d7eebbc --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/src/test/resources/starrocks-boolean-to-jdbc-tinyint.conf @@ -0,0 +1,48 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + StarRocks { + nodeUrls = ["starrocks_e2e:8030"] + username = root + password = "" + database = "test" + table = "e2e_boolean_as_tinyint_source" + schema { + fields { + id = INT + flag = TINYINT + } + } + } +} + +sink { + Jdbc { + driver = "com.mysql.cj.jdbc.Driver" + url = "jdbc:mysql://starrocks_e2e:9030/test" + username = root + password = "" + query = "INSERT INTO test.e2e_boolean_as_tinyint_sink (id, flag) VALUES (?, ?)" + batch_size = 3 + } +}
