This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/dev/pr-13143-19820c7e31f3ded2aa683446cb2bf396bcf65ef8 in repository https://gitbox.apache.org/repos/asf/seatunnel.git
commit 7ed5e28f4a49b367f0fbed80e1bd4e351cfb0776 Author: Ryan Yang <[email protected]> AuthorDate: Tue Nov 17 15:54:51 2026 +0000 [Fix][Connector-V2][File] Write NULL partition values instead of throwing NullPointerException (#13143) --- docs/en/connectors/sink/LocalFile.md | 2 + docs/zh/connectors/sink/LocalFile.md | 2 + .../file/sink/writer/AbstractWriteStrategy.java | 20 +++--- .../sink/writer/AbstractWriteStrategyTest.java | 73 ++++++++++++++++++++++ .../e2e/connector/file/local/LocalFileIT.java | 13 ++++ .../text/fake_to_local_file_null_partition.conf | 48 ++++++++++++++ 6 files changed, 149 insertions(+), 9 deletions(-) diff --git a/docs/en/connectors/sink/LocalFile.md b/docs/en/connectors/sink/LocalFile.md index b61a2e33f1..5a011151d1 100644 --- a/docs/en/connectors/sink/LocalFile.md +++ b/docs/en/connectors/sink/LocalFile.md @@ -152,6 +152,8 @@ Only used when `have_partition` is `true`. Partition data based on selected fields. +A `NULL` value in a partition field is written as the literal text `null` in the partition directory (for example `c_null=null`), not as a missing value. + ### partition_dir_expression [string] Only used when `have_partition` is `true`. diff --git a/docs/zh/connectors/sink/LocalFile.md b/docs/zh/connectors/sink/LocalFile.md index edfdb93ff3..10aada12c7 100644 --- a/docs/zh/connectors/sink/LocalFile.md +++ b/docs/zh/connectors/sink/LocalFile.md @@ -152,6 +152,8 @@ import ChangeLog from '../changelog/connector-file-local.md'; 基于选定字段进行数据分区。 +分区字段的值为 `NULL` 时,会以字面量 `null` 写入分区目录(例如 `c_null=null`),而不是作为缺失值处理。 + ### partition_dir_expression [string] 仅在 `have_partition` 为 `true` 时使用。 diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/AbstractWriteStrategy.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/AbstractWriteStrategy.java index 591391a82d..962b1f6aa9 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/AbstractWriteStrategy.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/AbstractWriteStrategy.java @@ -516,27 +516,29 @@ public abstract class AbstractWriteStrategy<T> implements WriteStrategy<T> { } List<String> vals = new ArrayList<>(partitionFieldsIndexInRow.size()); String partitionDir; + // A nullable partition column is a legal schema, so a null value must not fail the job: + // the directory name already renders null as the text "null", so the committed partition + // values use the same text instead of dereferencing the null field. if (StringUtils.isBlank(partitionDirExpression)) { StringBuilder stringBuilder = new StringBuilder(); for (int i = 0; i < partitionFieldsIndexInRow.size(); i++) { - stringBuilder - .append(partitionFieldList.get(i)) - .append("=") - .append(seaTunnelRow.getFields()[partitionFieldsIndexInRow.get(i)]); + String partitionValue = + String.valueOf(seaTunnelRow.getFields()[partitionFieldsIndexInRow.get(i)]); + stringBuilder.append(partitionFieldList.get(i)).append("=").append(partitionValue); if (i < partitionFieldsIndexInRow.size() - 1) { stringBuilder.append("/"); } - vals.add(seaTunnelRow.getFields()[partitionFieldsIndexInRow.get(i)].toString()); + vals.add(partitionValue); } partitionDir = stringBuilder.toString(); } else { Map<String, String> valueMap = new HashMap<>(partitionFieldList.size() * 2); for (int i = 0; i < partitionFieldsIndexInRow.size(); i++) { + String partitionValue = + String.valueOf(seaTunnelRow.getFields()[partitionFieldsIndexInRow.get(i)]); valueMap.put(keys[i], partitionFieldList.get(i)); - valueMap.put( - values[i], - seaTunnelRow.getFields()[partitionFieldsIndexInRow.get(i)].toString()); - vals.add(seaTunnelRow.getFields()[partitionFieldsIndexInRow.get(i)].toString()); + valueMap.put(values[i], partitionValue); + vals.add(partitionValue); } partitionDir = VariablesSubstitute.substitute(partitionDirExpression, valueMap); } diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/AbstractWriteStrategyTest.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/AbstractWriteStrategyTest.java index 06ef8451a5..6d468f5078 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/AbstractWriteStrategyTest.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/AbstractWriteStrategyTest.java @@ -42,8 +42,11 @@ import org.mockito.Mockito; import java.io.File; import java.io.IOException; import java.io.OutputStream; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; @@ -177,6 +180,56 @@ class AbstractWriteStrategyTest { Assertions.assertTrue(deleteCalled.await(5, TimeUnit.SECONDS)); } + @Test + void generatorPartitionDirWithNullPartitionValue() { + TestTextWriteStrategy writeStrategy = + new TestTextWriteStrategy( + newPartitionedFileSinkConfig(newPartitionRowType(), null)); + + LinkedHashMap<String, List<String>> partitionDirAndValues = + writeStrategy.generatorPartitionDir(new SeaTunnelRow(new Object[] {1, null})); + + Assertions.assertEquals( + Collections.singletonList("c_null=null"), + new ArrayList<>(partitionDirAndValues.keySet())); + Assertions.assertEquals( + Collections.singletonList("null"), partitionDirAndValues.get("c_null=null")); + } + + @Test + void generatorPartitionDirWithNullPartitionValueAndExpression() { + // "${k0}/${v0}" renders differently from the default branch on purpose: if the expression + // were ignored, the directory would be "c_null=null" and the assertion below would fail. + TestTextWriteStrategy writeStrategy = + new TestTextWriteStrategy( + newPartitionedFileSinkConfig(newPartitionRowType(), "${k0}/${v0}")); + + LinkedHashMap<String, List<String>> partitionDirAndValues = + writeStrategy.generatorPartitionDir(new SeaTunnelRow(new Object[] {1, null})); + + Assertions.assertEquals( + Collections.singletonList("c_null/null"), + new ArrayList<>(partitionDirAndValues.keySet())); + Assertions.assertEquals( + Collections.singletonList("null"), partitionDirAndValues.get("c_null/null")); + } + + @Test + void generatorPartitionDirKeepsNonNullPartitionValue() { + TestTextWriteStrategy writeStrategy = + new TestTextWriteStrategy( + newPartitionedFileSinkConfig(newPartitionRowType(), null)); + + LinkedHashMap<String, List<String>> partitionDirAndValues = + writeStrategy.generatorPartitionDir(new SeaTunnelRow(new Object[] {1, "alice"})); + + Assertions.assertEquals( + Collections.singletonList("c_null=alice"), + new ArrayList<>(partitionDirAndValues.keySet())); + Assertions.assertEquals( + Collections.singletonList("alice"), partitionDirAndValues.get("c_null=alice")); + } + @Test void shouldNotCloseOrcWriterWhileWriteIsInProgress() throws Exception { assertCloseWaitsForColumnarWrite(Mockito.spy(new OrcWriteStrategy(newFileSinkConfig()))); @@ -258,6 +311,26 @@ class AbstractWriteStrategyTest { return fileSinkConfig; } + private static SeaTunnelRowType newPartitionRowType() { + return new SeaTunnelRowType( + new String[] {"id", "c_null"}, + new SeaTunnelDataType[] {BasicType.INT_TYPE, BasicType.STRING_TYPE}); + } + + private static FileSinkConfig newPartitionedFileSinkConfig( + SeaTunnelRowType rowType, String partitionDirExpression) { + Map<String, Object> writeConfig = new HashMap<>(); + writeConfig.put("tmp_path", TMP_PATH); + writeConfig.put("path", "/tmp/seatunnel-target"); + writeConfig.put("file_format_type", FileFormat.TEXT.name()); + writeConfig.put("partition_by", Collections.singletonList("c_null")); + if (partitionDirExpression != null) { + writeConfig.put("partition_dir_expression", partitionDirExpression); + } + return new FileSinkConfig( + ReadonlyConfig.fromConfig(ConfigFactory.parseMap(writeConfig)), rowType); + } + private static FileSinkConfig newTextFileSinkConfig(SeaTunnelRowType rowType) { Map<String, Object> writeConfig = new HashMap<>(); writeConfig.put("tmp_path", TMP_PATH); diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileIT.java index a8aa98c614..998c5f7d89 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileIT.java @@ -428,6 +428,19 @@ public class LocalFileIT extends TestSuiteBase { helper.execute("/text/local_file_null_format_assert.conf"); } + @TestTemplate + public void testLocalFileNullPartitionValue(TestContainer container) + throws IOException, InterruptedException { + TestHelper helper = new TestHelper(container); + // A NULL partition value used to fail the job with a NullPointerException. The job must + // now finish and write the partition directory with the literal text "null". + helper.execute("/text/fake_to_local_file_null_partition.conf"); + Container.ExecResult execResult = + baseContainer.execInContainer( + "sh", "-c", "ls -d /tmp/seatunnel/text_null_partition/c_null=null"); + Assertions.assertEquals(0, execResult.getExitCode(), execResult.getStderr()); + } + @TestTemplate public void testLocalFileJson(TestContainer container) throws IOException, InterruptedException { diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/text/fake_to_local_file_null_partition.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/text/fake_to_local_file_null_partition.conf new file mode 100644 index 0000000000..87ea5f8b4a --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/text/fake_to_local_file_null_partition.conf @@ -0,0 +1,48 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + FakeSource { + row.num = 10 + plugin_output = "fake" + schema = { + fields { + id = int + c_null = "null" + } + } + } +} + +sink { + LocalFile { + plugin_input = "fake" + path = "/tmp/seatunnel/text_null_partition" + field_delimiter = "\t" + row_delimiter = "\n" + partition_by = ["c_null"] + file_name_expression = "${transactionId}_${now}" + file_format_type = "text" + filename_time_format = "yyyy.MM.dd" + is_enable_transaction = true + } +}
