This is an automated email from the ASF dual-hosted git repository. gaojun2048 pushed a commit to branch revert-5427-fix-synchive-empty-parition in repository https://gitbox.apache.org/repos/asf/seatunnel.git
commit 4e2f4ef1169ad66eed3f4351288888366d5b01df Author: Eric <[email protected]> AuthorDate: Thu Sep 14 19:41:41 2023 +0800 Revert "[fix][hive-source][bug] fix An error occurred reading an empty directory (#5427)" This reverts commit de7b86a5dd24d9cff4fa28ca7b8eadbd50ed8bb9. --- .../file/source/reader/AbstractReadStrategy.java | 22 ++++++++++++++-------- .../file/source/reader/ExcelReadStrategy.java | 3 +-- .../file/source/reader/TextReadStrategy.java | 3 +-- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategy.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategy.java index 1dc8e7a2ad..e4e1694f30 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategy.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategy.java @@ -24,6 +24,8 @@ import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.api.table.type.SeaTunnelRowType; import org.apache.seatunnel.connectors.seatunnel.file.config.BaseSourceConfig; import org.apache.seatunnel.connectors.seatunnel.file.config.HadoopConf; +import org.apache.seatunnel.connectors.seatunnel.file.exception.FileConnectorErrorCode; +import org.apache.seatunnel.connectors.seatunnel.file.exception.FileConnectorException; import org.apache.seatunnel.connectors.seatunnel.file.sink.util.FileSystemUtils; import org.apache.hadoop.conf.Configuration; @@ -151,9 +153,15 @@ public abstract class AbstractReadStrategy implements ReadStrategy { } } } - if (this.fileNames.isEmpty()) { - log.error("The current directory is empty " + path); + + if (fileNames.isEmpty()) { + throw new FileConnectorException( + FileConnectorErrorCode.FILE_LIST_EMPTY, + "The target file list is empty," + + "SeaTunnel will not be able to sync empty table, " + + "please check the configuration parameters such as: [file_filter_pattern]"); } + return fileNames; } @@ -188,12 +196,10 @@ public abstract class AbstractReadStrategy implements ReadStrategy { protected Map<String, String> parsePartitionsByPath(String path) { LinkedHashMap<String, String> partitions = new LinkedHashMap<>(); - if (path != null && !path.isEmpty()) { - Arrays.stream(path.split("/", -1)) - .filter(split -> split.contains("=")) - .map(split -> split.split("=", -1)) - .forEach(kv -> partitions.put(kv[0], kv[1])); - } + Arrays.stream(path.split("/", -1)) + .filter(split -> split.contains("=")) + .map(split -> split.split("=", -1)) + .forEach(kv -> partitions.put(kv[0], kv[1])); return partitions; } diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/ExcelReadStrategy.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/ExcelReadStrategy.java index 649216c752..0b1cfc083b 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/ExcelReadStrategy.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/ExcelReadStrategy.java @@ -136,8 +136,7 @@ public class ExcelReadStrategy extends AbstractReadStrategy { "Schmea information is not set or incorrect schmea settings"); } SeaTunnelRowType userDefinedRowTypeWithPartition = - mergePartitionTypes( - fileNames.size() > 0 ? fileNames.get(0) : null, seaTunnelRowType); + mergePartitionTypes(fileNames.get(0), seaTunnelRowType); // column projection if (pluginConfig.hasPath(BaseSourceConfig.READ_COLUMNS.key())) { // get the read column index from user-defined row type diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/TextReadStrategy.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/TextReadStrategy.java index eefa8e4de7..4b931cb890 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/TextReadStrategy.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/TextReadStrategy.java @@ -138,8 +138,7 @@ public class TextReadStrategy extends AbstractReadStrategy { @Override public void setSeaTunnelRowTypeInfo(SeaTunnelRowType seaTunnelRowType) { SeaTunnelRowType userDefinedRowTypeWithPartition = - mergePartitionTypes( - fileNames.size() > 0 ? fileNames.get(0) : null, seaTunnelRowType); + mergePartitionTypes(fileNames.get(0), seaTunnelRowType); if (pluginConfig.hasPath(BaseSourceConfig.DELIMITER.key())) { fieldDelimiter = pluginConfig.getString(BaseSourceConfig.DELIMITER.key()); } else {
