This is an automated email from the ASF dual-hosted git repository.
davidzollo pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 261644ff40 [Improve][Connector-V2] Guard POI Excel reads by file size
(#11591)
261644ff40 is described below
commit 261644ff409ce4340c58edcda8dbcb2b2c5cf071
Author: Jast <[email protected]>
AuthorDate: Tue Aug 11 22:50:03 2026 +0800
[Improve][Connector-V2] Guard POI Excel reads by file size (#11591)
Signed-off-by: Shenghang Zhang <[email protected]>
Co-authored-by: zhangshenghang <[email protected]>
Co-authored-by: zhangshenghang <[email protected]>
Co-authored-by: Claude Fable 5 <[email protected]>
---
docs/en/connectors/source/CosFile.md | 18 ++
docs/en/connectors/source/FtpFile.md | 18 ++
docs/en/connectors/source/HdfsFile.md | 18 ++
docs/en/connectors/source/LocalFile.md | 11 +-
docs/en/connectors/source/ObsFile.md | 2 +
docs/en/connectors/source/OssFile.md | 2 +
docs/en/connectors/source/OssJindoFile.md | 18 ++
docs/en/connectors/source/S3File.md | 2 +
docs/en/connectors/source/SftpFile.md | 2 +
.../introduction/concepts/incompatible-changes.md | 6 +
docs/zh/connectors/source/CosFile.md | 18 ++
docs/zh/connectors/source/FtpFile.md | 18 ++
docs/zh/connectors/source/HdfsFile.md | 18 ++
docs/zh/connectors/source/LocalFile.md | 13 +-
docs/zh/connectors/source/ObsFile.md | 3 +
docs/zh/connectors/source/OssFile.md | 2 +
docs/zh/connectors/source/OssJindoFile.md | 2 +
docs/zh/connectors/source/S3File.md | 2 +
docs/zh/connectors/source/SftpFile.md | 2 +
.../introduction/concepts/incompatible-changes.md | 6 +
.../seatunnel/file/config/FileBaseOptions.java | 10 +-
.../file/config/FileBaseSourceOptions.java | 10 +-
.../file/source/reader/AbstractReadStrategy.java | 75 ++++-
.../file/source/reader/ExcelReadStrategy.java | 307 +++++++++++++++------
.../file/reader/ExcelReadStrategyTest.java | 204 ++++++++++++++
.../resources/excel/archive_zip/e2e_in_zip.zip | Bin 0 -> 5324 bytes
.../file/cos/source/CosFileSourceFactory.java | 8 +-
.../file/ftp/source/FtpFileSourceFactory.java | 4 +
.../file/hdfs/source/HdfsFileSourceFactory.java | 4 +
.../oss/jindo/source/OssFileSourceFactory.java | 4 +
.../file/local/source/LocalFileSourceFactory.java | 4 +
.../file/obs/source/ObsFileSourceFactory.java | 4 +
.../file/oss/source/OssFileSourceFactory.java | 4 +
.../file/s3/source/S3FileSourceFactory.java | 4 +
.../file/sftp/source/SftpFileSourceFactory.java | 4 +
35 files changed, 728 insertions(+), 99 deletions(-)
diff --git a/docs/en/connectors/source/CosFile.md
b/docs/en/connectors/source/CosFile.md
index 857ad70902..75439fac3f 100644
--- a/docs/en/connectors/source/CosFile.md
+++ b/docs/en/connectors/source/CosFile.md
@@ -71,6 +71,8 @@ To use this connector you need put
hadoop-cos-{hadoop.version}-{version}.jar and
| time_format | string | no | HH:mm:ss
|
| schema | config | no | -
|
| sheet_name | string | no | -
|
+| excel_engine | string | no | POI
|
+| poi_excel_max_file_size | long | no | 52428800
|
| xml_row_tag | string | no | -
|
| xml_use_attr_format | boolean | no | -
|
| csv_use_header_line | boolean | no | false
|
@@ -323,6 +325,22 @@ Only need to be configured when file_format is excel.
Reader the sheet of the workbook.
+### excel_engine [string]
+
+Only used when `file_format` is excel.
+
+Supported engines are `POI` and `EasyExcel`. The default value is `POI`.
+
+The default Excel reading engine is POI. POI keeps the historical read
behavior, including POI-specific formula and formatting handling, but it may
use a lot of memory for large Excel files.
+
+You can set `excel_engine = EasyExcel` to use streaming reads for large Excel
files.
+
+### poi_excel_max_file_size [long]
+
+Only used when `file_format` is excel and `excel_engine` is POI.
+
+The maximum Excel file size in bytes that the POI engine can read. The default
value is `52428800` bytes (50 MB). When the file is larger than this limit, the
connector fails fast and suggests using EasyExcel.
+
### xml_row_tag [string]
Only need to be configured when file_format is xml.
diff --git a/docs/en/connectors/source/FtpFile.md
b/docs/en/connectors/source/FtpFile.md
index c98c44119a..10c94e8498 100644
--- a/docs/en/connectors/source/FtpFile.md
+++ b/docs/en/connectors/source/FtpFile.md
@@ -69,6 +69,8 @@ If you use SeaTunnel Engine, It automatically integrated the
hadoop jar when you
| skip_header_row_number | long | no | 0
|
| schema | config | no | -
|
| sheet_name | string | no | -
|
+| excel_engine | string | no | POI
|
+| poi_excel_max_file_size | long | no | 52428800
|
| xml_row_tag | string | no | -
|
| xml_use_attr_format | boolean | no | -
|
| csv_use_header_line | boolean | no | -
|
@@ -430,6 +432,22 @@ The read column list of the data source, user can use it
to implement field proj
Reader the sheet of the workbook,Only used when file_format_type is excel.
+### excel_engine [string]
+
+Only used when `file_format` is excel.
+
+Supported engines are `POI` and `EasyExcel`. The default value is `POI`.
+
+The default Excel reading engine is POI. POI keeps the historical read
behavior, including POI-specific formula and formatting handling, but it may
use a lot of memory for large Excel files.
+
+You can set `excel_engine = EasyExcel` to use streaming reads for large Excel
files.
+
+### poi_excel_max_file_size [long]
+
+Only used when `file_format` is excel and `excel_engine` is POI.
+
+The maximum Excel file size in bytes that the POI engine can read. The default
value is `52428800` bytes (50 MB). When the file is larger than this limit, the
connector fails fast and suggests using EasyExcel.
+
### xml_row_tag [string]
Only need to be configured when file_format is xml.
diff --git a/docs/en/connectors/source/HdfsFile.md
b/docs/en/connectors/source/HdfsFile.md
index 3b2609ca81..ec29badc49 100644
--- a/docs/en/connectors/source/HdfsFile.md
+++ b/docs/en/connectors/source/HdfsFile.md
@@ -71,6 +71,8 @@ Read data from hdfs file system.
| skip_header_row_number | long | no | 0
| Skip the first few lines, but only for the txt and csv.For example, set like
following:`skip_header_row_number = 2`.then Seatunnel will skip the first 2
lines from source files
|
| schema | config | no | -
| the schema fields of upstream data. For more details, please refer to
[Schema Feature](../../introduction/concepts/schema-feature.md).
**metadata_table_id**: The table identifier in the metadata service to fetch
table schema. For Gravitino, the format should be
`{catalog}.{database}.{table}`, such as `mysql-catalog.test_db.users`. When
using Gravitino as the metadata source, the column types from Gravitino [...]
| sheet_name | string | no | -
| Reader the sheet of the workbook,Only used when file_format is excel.
|
+| excel_engine | string | no | POI
| Only used when `file_format` is excel. Supported engines are `POI` and
`EasyExcel`.
|
+| poi_excel_max_file_size | long | no | 52428800
| Only used when `file_format` is excel and `excel_engine` is POI. The maximum
Excel file size in bytes that the POI engine can read (default 50 MB).
|
| xml_row_tag | string | no | -
| Specifies the tag name of the data rows within the XML file, only used when
file_format is xml.
|
| xml_use_attr_format | boolean | no | -
| Specifies whether to process data using the tag attribute format, only used
when file_format is xml.
|
| csv_use_header_line | boolean | no | false
| Whether to use the header line to parse the file, only used when the
file_format is `csv` and the file contains the header line that match RFC 4180
|
@@ -221,6 +223,22 @@ The result of this example matching is:
/data/seatunnel/20241005/old_data.csv
```
+### excel_engine [string]
+
+Only used when `file_format` is excel.
+
+Supported engines are `POI` and `EasyExcel`. The default value is `POI`.
+
+The default Excel reading engine is POI. POI keeps the historical read
behavior, including POI-specific formula and formatting handling, but it may
use a lot of memory for large Excel files.
+
+You can set `excel_engine = EasyExcel` to use streaming reads for large Excel
files.
+
+### poi_excel_max_file_size [long]
+
+Only used when `file_format` is excel and `excel_engine` is POI.
+
+The maximum Excel file size in bytes that the POI engine can read. The default
value is `52428800` bytes (50 MB). When the file is larger than this limit, the
connector fails fast and suggests using EasyExcel.
+
### compress_codec [string]
The compress codec of files and the details that supported as the following
shown:
diff --git a/docs/en/connectors/source/LocalFile.md
b/docs/en/connectors/source/LocalFile.md
index 7959a1b184..0a327fe104 100644
--- a/docs/en/connectors/source/LocalFile.md
+++ b/docs/en/connectors/source/LocalFile.md
@@ -66,6 +66,7 @@ If you use SeaTunnel Engine, It automatically integrated the
hadoop jar when you
| schema | config | no | -
|
| sheet_name | string | no | -
|
| excel_engine | string | no | POI
|
+| poi_excel_max_file_size | long | no | 52428800
|
| xml_row_tag | string | no | -
|
| xml_use_attr_format | boolean | no | -
|
| csv_use_header_line | boolean | no | false
|
@@ -338,7 +339,15 @@ Only need to be configured when file_format is excel.
supported as the following file types:
`POI` `EasyExcel`
-The default excel reading engine is POI, but POI can easily cause memory
overflow when reading Excel with more than 65,000 rows, so you can switch to
EasyExcel as the reading engine.
+The default Excel reading engine is POI. POI keeps the historical read
behavior, including POI-specific formula and formatting handling, but it may
use a lot of memory for large Excel files.
+
+You can set `excel_engine = EasyExcel` to use streaming reads for large Excel
files.
+
+### poi_excel_max_file_size [long]
+
+Only used when `file_format` is excel and `excel_engine` is POI.
+
+The maximum Excel file size in bytes that the POI engine can read. The default
value is `52428800` bytes (50 MB). When the file is larger than this limit, the
connector fails fast and suggests using EasyExcel.
### xml_row_tag [string]
diff --git a/docs/en/connectors/source/ObsFile.md
b/docs/en/connectors/source/ObsFile.md
index 60aba06bf0..29fd34ac17 100644
--- a/docs/en/connectors/source/ObsFile.md
+++ b/docs/en/connectors/source/ObsFile.md
@@ -83,6 +83,8 @@ It only supports hadoop version **2.9.X+**.
| schema | config | no | - |
[Tips](#schema)
|
| common-options | | no | - |
[Tips](#common_options)
|
| sheet_name | string | no | - |
Reader the sheet of the workbook,Only used when file_format is excel.
|
+| excel_engine | string | no | POI | Only
used when `file_format` is excel. Supported engines are `POI` and `EasyExcel`.
|
+| poi_excel_max_file_size | long | no | 52428800 | Only
used when `file_format` is excel and `excel_engine` is POI. The maximum Excel
file size in bytes that the POI engine can read (default 50 MB).
|
| file_filter_modified_start | string | no | - | File
modification time filter. The connector will filter some files base on the last
modification start time (include start time). The default data format is
`yyyy-MM-dd HH:mm:ss`. |
| file_filter_modified_end | string | no | - | File
modification time filter. The connector will filter some files base on the last
modification end time (not include end time). The default data format is
`yyyy-MM-dd HH:mm:ss`. |
| quote_char | string | no | " | A
single character that encloses CSV fields, allowing fields with commas, line
breaks, or quotes to be read correctly.
|
diff --git a/docs/en/connectors/source/OssFile.md
b/docs/en/connectors/source/OssFile.md
index 5c4e6fcc8d..08275a88a7 100644
--- a/docs/en/connectors/source/OssFile.md
+++ b/docs/en/connectors/source/OssFile.md
@@ -203,6 +203,8 @@ If you assign file type to `parquet` `orc`, schema option
not required, connecto
| skip_header_row_number | long | no | 0 | Skip
the first few lines, but only for the txt and csv. For example, set like
following:`skip_header_row_number = 2`. Then SeaTunnel will skip the first 2
lines from source files
|
| schema | config | no | - | The
schema of upstream data.
|
| sheet_name | string | no | - |
Reader the sheet of the workbook,Only used when file_format is excel.
|
+| excel_engine | string | no | POI | Only
used when `file_format` is excel. Supported engines are `POI` and `EasyExcel`.
|
+| poi_excel_max_file_size | long | no | 52428800 | Only
used when `file_format` is excel and `excel_engine` is POI. The maximum Excel
file size in bytes that the POI engine can read (default 50 MB).
|
| xml_row_tag | string | no | - |
Specifies the tag name of the data rows within the XML file, only used when
file_format is xml.
|
| xml_use_attr_format | boolean | no | - |
Specifies whether to process data using the tag attribute format, only used
when file_format is xml.
|
| csv_use_header_line | boolean | no | false |
Whether to use the header line to parse the file, only used when the
file_format is `csv` and the file contains the header line that match RFC 4180
|
diff --git a/docs/en/connectors/source/OssJindoFile.md
b/docs/en/connectors/source/OssJindoFile.md
index 55858e8c94..1d05a53422 100644
--- a/docs/en/connectors/source/OssJindoFile.md
+++ b/docs/en/connectors/source/OssJindoFile.md
@@ -75,6 +75,8 @@ It only supports hadoop version **2.9.X+**.
| skip_header_row_number | long | no | 0
|
| schema | config | no | -
|
| sheet_name | string | no | -
|
+| excel_engine | string | no | POI
|
+| poi_excel_max_file_size | long | no | 52428800
|
| xml_row_tag | string | no | -
|
| xml_use_attr_format | boolean | no | -
|
| csv_use_header_line | boolean | no | false
|
@@ -325,6 +327,22 @@ Only need to be configured when file_format is excel.
Reader the sheet of the workbook.
+### excel_engine [string]
+
+Only used when `file_format` is excel.
+
+Supported engines are `POI` and `EasyExcel`. The default value is `POI`.
+
+The default Excel reading engine is POI. POI keeps the historical read
behavior, including POI-specific formula and formatting handling, but it may
use a lot of memory for large Excel files.
+
+You can set `excel_engine = EasyExcel` to use streaming reads for large Excel
files.
+
+### poi_excel_max_file_size [long]
+
+Only used when `file_format` is excel and `excel_engine` is POI.
+
+The maximum Excel file size in bytes that the POI engine can read. The default
value is `52428800` bytes (50 MB). When the file is larger than this limit, the
connector fails fast and suggests using EasyExcel.
+
### file_filter_pattern [string]
Filter pattern, which used for filtering files. If you only want to filter
based on file names, simply write the regular file names; If you want to filter
based on the file directory at the same time, the expression needs to start
with `path`.
diff --git a/docs/en/connectors/source/S3File.md
b/docs/en/connectors/source/S3File.md
index de51d8b9bd..c9a7003382 100644
--- a/docs/en/connectors/source/S3File.md
+++ b/docs/en/connectors/source/S3File.md
@@ -211,6 +211,8 @@ If you assign file type to `parquet` `orc`, schema option
not required, connecto
| csv_use_header_line | boolean | no | false
| Whether to use the header line to parse the
file, only used when the file_format is `csv` and the file contains the header
line that match RFC 4180
[...]
| schema | config | no | -
| The schema of upstream data. For more
details, please refer to [Schema
Feature](../../introduction/concepts/schema-feature.md).
[...]
| sheet_name | string | no | -
| Reader the sheet of the workbook,Only used
when file_format is excel.
[...]
+| excel_engine | string | no | POI
| Only used when `file_format` is excel.
Supported engines are `POI` and `EasyExcel`.
[...]
+| poi_excel_max_file_size | long | no | 52428800
| Only used when `file_format` is excel and
`excel_engine` is POI. The maximum Excel file size in bytes that the POI engine
can read (default 50 MB).
[...]
| xml_row_tag | string | no | -
| Specifies the tag name of the data rows
within the XML file, only valid for XML files.
[...]
| xml_use_attr_format | boolean | no | -
| Specifies whether to process data using the
tag attribute format, only valid for XML files.
[...]
| csv_use_header_line | boolean | no | false
| Whether to use the header line to parse the
file, only used when the file_format is `csv` and the file contains the header
line that match RFC 4180
[...]
diff --git a/docs/en/connectors/source/SftpFile.md
b/docs/en/connectors/source/SftpFile.md
index b6c71f8ff0..f3acf6bdd6 100644
--- a/docs/en/connectors/source/SftpFile.md
+++ b/docs/en/connectors/source/SftpFile.md
@@ -99,6 +99,8 @@ The File does not have a specific type list, and we can
indicate which SeaTunnel
| skip_header_row_number | Long | No | 0
| Skip the first few lines, but only for the txt and csv. <br/> For example,
set like following: <br/> `skip_header_row_number = 2` <br/> then SeaTunnel
will skip the first 2 lines from source files
|
| read_columns | list | no | -
| The read column list of the data source, user can use it to implement
field projection.
|
| sheet_name | String | No | -
| Reader the sheet of the workbook,Only used when file_format is excel.
|
+| excel_engine | string | no | POI
| Only used when `file_format` is excel. Supported engines are `POI` and
`EasyExcel`.
|
+| poi_excel_max_file_size | long | no | 52428800
| Only used when `file_format` is excel and `excel_engine` is POI. The
maximum Excel file size in bytes that the POI engine can read (default 50 MB).
|
| xml_row_tag | string | no | -
| Specifies the tag name of the data rows within the XML file, only used
when file_format is xml.
|
| xml_use_attr_format | boolean | no | -
| Specifies whether to process data using the tag attribute format, only
used when file_format is xml.
|
| csv_use_header_line | boolean | no | false
| Whether to use the header line to parse the file, only used when the
file_format is `csv` and the file contains the header line that match RFC 4180
|
diff --git a/docs/en/introduction/concepts/incompatible-changes.md
b/docs/en/introduction/concepts/incompatible-changes.md
index 5cce8d2a8d..d1d3159d42 100644
--- a/docs/en/introduction/concepts/incompatible-changes.md
+++ b/docs/en/introduction/concepts/incompatible-changes.md
@@ -114,6 +114,12 @@ You need to check this document before you upgrade to
related version.
Glue/Hive metastore schema are not affected at runtime; only newly
auto-created tables change
behavior.
+- **Breaking Change: File source connectors reject POI-engine Excel files
larger than `poi_excel_max_file_size` (default 50 MB)**
+ - **Affected component**: `seatunnel-connectors-v2/connector-file`
(LocalFile, HdfsFile, S3File, FtpFile, SftpFile, OssFile, OssJindoFile,
ObsFile, CosFile)
+ - **Description**: Apache POI fully materializes an Excel workbook into
memory before any row can be read, which can drive a Zeta worker into heavy GC
pressure or OOM on large `.xls`/`.xlsx` files. A new `poi_excel_max_file_size`
option (default 50 MB) now makes POI reject an Excel file that exceeds the
limit before the workbook is built. The guard covers both plain and archived
(ZIP/TAR/TAR_GZ/GZ) Excel entries, and applies only when `excel_engine = POI`
(the default); the streaming ` [...]
+ - **Impact**: Existing jobs that read POI-engine Excel files larger than 50
MB - which previously succeeded at the cost of heavy memory pressure - will now
fail fast with a `FileConnectorException` instead of potentially OOMing the
worker.
+ - **Migration Guide**: For POI jobs that must read large Excel files and
have sufficient worker memory, raise the limit with `poi_excel_max_file_size =
<bytes>`. Otherwise switch to `excel_engine = EasyExcel`, which streams rows
lazily and is not subject to the limit.
+
### Transform Changes
- **[BREAKING]** SQL Transform `PARSEDATETIME`, `TO_DATE`, and `IS_DATE`
functions now only accept whitelisted datetime format patterns. Custom format
patterns that were previously accepted will now fail at runtime. The supported
patterns are:
diff --git a/docs/zh/connectors/source/CosFile.md
b/docs/zh/connectors/source/CosFile.md
index 68d5faf90c..a9e857efa5 100644
--- a/docs/zh/connectors/source/CosFile.md
+++ b/docs/zh/connectors/source/CosFile.md
@@ -71,6 +71,8 @@ import ChangeLog from '../changelog/connector-file-cos.md';
| time_format | string | 否 | HH:mm:ss |
| schema | config | 否 | - |
| sheet_name | string | 否 | - |
+| excel_engine | string | 否 | POI |
+| poi_excel_max_file_size | long | 否 | 52428800 |
| xml_row_tag | string | 否 | - |
| xml_use_attr_format | boolean | 否 | - |
| csv_use_header_line | boolean | 否 | false |
@@ -323,6 +325,22 @@ default `HH:mm:ss`
阅读工作簿的纸张。
+### excel_engine [string]
+
+仅在 `file_format` 为 excel 时使用。
+
+支持的引擎包括 `POI` 和 `EasyExcel`。默认值为 `POI`。
+
+默认的 Excel 读取引擎是 POI。POI 会保留历史读取行为，包括 POI 特有的公式和格式处理能力，但读取大 Excel 文件时可能占用大量内存。
+
+如果需要读取大 Excel 文件，可以设置 `excel_engine = EasyExcel` 使用流式读取。
+
+### poi_excel_max_file_size [long]
+
+仅在 `file_format` 为 excel 且 `excel_engine` 为 POI 时使用。
+
+POI 引擎允许读取的最大 Excel 文件大小，单位为字节。默认值为 `52428800` 字节（50
MB）。当文件超过该限制时，连接器会提前失败，并提示使用 EasyExcel。
+
### xml_row_tag [string]
仅当file_format为xml时才需要配置。
diff --git a/docs/zh/connectors/source/FtpFile.md
b/docs/zh/connectors/source/FtpFile.md
index 306e786231..3dd50e53f1 100644
--- a/docs/zh/connectors/source/FtpFile.md
+++ b/docs/zh/connectors/source/FtpFile.md
@@ -68,6 +68,8 @@ import ChangeLog from '../changelog/connector-file-ftp.md';
| skip_header_row_number | long | 否 | 0 |
| schema | config | 否 | - |
| sheet_name | string | 否 | - |
+| excel_engine | string | 否 | POI |
+| poi_excel_max_file_size | long | 否 | 52428800 |
| xml_row_tag | string | 否 | - |
| xml_use_attr_format | boolean | 否 | - |
| csv_use_header_line | boolean | 否 | false |
@@ -402,6 +404,22 @@ SeaTunnel 将从源文件中跳过前 2 行。
读取工作簿中的工作表，仅在文件格式类型为 excel 时使用。
+### excel_engine [string]
+
+仅在 `file_format` 为 excel 时使用。
+
+支持的引擎包括 `POI` 和 `EasyExcel`。默认值为 `POI`。
+
+默认的 Excel 读取引擎是 POI。POI 会保留历史读取行为，包括 POI 特有的公式和格式处理能力，但读取大 Excel 文件时可能占用大量内存。
+
+如果需要读取大 Excel 文件，可以设置 `excel_engine = EasyExcel` 使用流式读取。
+
+### poi_excel_max_file_size [long]
+
+仅在 `file_format` 为 excel 且 `excel_engine` 为 POI 时使用。
+
+POI 引擎允许读取的最大 Excel 文件大小，单位为字节。默认值为 `52428800` 字节（50
MB）。当文件超过该限制时，连接器会提前失败，并提示使用 EasyExcel。
+
### xml_row_tag [string]
仅在文件格式为 xml 时需要配置。
diff --git a/docs/zh/connectors/source/HdfsFile.md
b/docs/zh/connectors/source/HdfsFile.md
index 5f155c4495..400ccb5077 100644
--- a/docs/zh/connectors/source/HdfsFile.md
+++ b/docs/zh/connectors/source/HdfsFile.md
@@ -71,6 +71,8 @@ import ChangeLog from '../changelog/connector-file-hadoop.md';
| skip_header_row_number | long | 否 | 0 |
跳过前几行，但仅适用于 txt 和 csv。例如，设置如下：`skip_header_row_number = 2`。然后 Seatunnel
将跳过源文件的前 2 行
|
| schema | config | 否 | - | 上游数据的
schema 字段。更多详情请参考 [Schema 特性](../../introduction/concepts/schema-feature.md)。
|
| sheet_name | string | 否 | - |
读取工作簿的工作表，仅在 file_format 为 excel 时使用。
|
+| excel_engine | string | 否 | POI | 仅在
`file_format` 为 excel 时使用。支持的引擎包括 `POI` 和 `EasyExcel`。
|
+| poi_excel_max_file_size | long | 否 | 52428800 | 仅在
`file_format` 为 excel 且 `excel_engine` 为 POI 时使用。POI 引擎允许读取的最大 Excel 文件大小（默认 50
MB）。
|
| xml_row_tag | string | 否 | - | 指定 XML
文件中数据行的标签名称，仅在 file_format 为 xml 时使用。
|
| xml_use_attr_format | boolean | 否 | - |
指定是否使用标签属性格式处理数据，仅在 file_format 为 xml 时使用。
|
| csv_use_header_line | boolean | 否 | false |
是否使用标题行解析文件，仅在 file_format 为 `csv` 且文件包含符合 RFC 4180 的标题行时使用
|
@@ -221,6 +223,22 @@ abc.*
/data/seatunnel/20241005/old_data.csv
```
+### excel_engine [string]
+
+仅在 `file_format` 为 excel 时使用。
+
+支持的引擎包括 `POI` 和 `EasyExcel`。默认值为 `POI`。
+
+默认的 Excel 读取引擎是 POI。POI 会保留历史读取行为，包括 POI 特有的公式和格式处理能力，但读取大 Excel 文件时可能占用大量内存。
+
+如果需要读取大 Excel 文件，可以设置 `excel_engine = EasyExcel` 使用流式读取。
+
+### poi_excel_max_file_size [long]
+
+仅在 `file_format` 为 excel 且 `excel_engine` 为 POI 时使用。
+
+POI 引擎允许读取的最大 Excel 文件大小，单位为字节。默认值为 `52428800` 字节（50
MB）。当文件超过该限制时，连接器会提前失败，并提示使用 EasyExcel。
+
### compress_codec [string]
文件的压缩编解码器及其支持的详细信息如下所示：
diff --git a/docs/zh/connectors/source/LocalFile.md
b/docs/zh/connectors/source/LocalFile.md
index f0c9ee9ddd..d318798fe0 100644
--- a/docs/zh/connectors/source/LocalFile.md
+++ b/docs/zh/connectors/source/LocalFile.md
@@ -65,7 +65,8 @@ import ChangeLog from '../changelog/connector-file-local.md';
| skip_header_row_number | long | 否 | 0 |
| schema | config | 否 | - |
| sheet_name | string | 否 | - |
-| excel_engine | string | 否 | POI |
+| excel_engine | string | 否 | POI |
+| poi_excel_max_file_size | long | 否 | 52428800 |
| xml_row_tag | string | 否 | - |
| xml_use_attr_format | boolean | 否 | - |
| csv_use_header_line | boolean | 否 | false |
@@ -338,7 +339,15 @@ PDF 特有的解析行为如下：
支持以下文件类型：
`POI` `EasyExcel`
-默认的 excel 读取引擎是 POI，但当读取超过 65,000 行的 Excel 时，POI 容易导致内存溢出，因此您可以切换到 EasyExcel
作为读取引擎。
+默认的 Excel 读取引擎是 POI。POI 会保留历史读取行为，包括 POI 特有的公式和格式处理能力，但读取大 Excel 文件时可能占用大量内存。
+
+如果需要读取大 Excel 文件，可以设置 `excel_engine = EasyExcel` 使用流式读取。
+
+### poi_excel_max_file_size [long]
+
+仅在 `file_format` 为 excel 且 `excel_engine` 为 POI 时使用。
+
+POI 引擎允许读取的最大 Excel 文件大小，单位为字节。默认值为 `52428800` 字节（50
MB）。当文件超过该限制时，连接器会提前失败，并提示使用 EasyExcel。
### xml_row_tag [string]
diff --git a/docs/zh/connectors/source/ObsFile.md
b/docs/zh/connectors/source/ObsFile.md
index fbfeaaaf57..5644c3236f 100644
--- a/docs/zh/connectors/source/ObsFile.md
+++ b/docs/zh/connectors/source/ObsFile.md
@@ -72,6 +72,9 @@ import ChangeLog from '../changelog/connector-file-obs.md';
| access_secret | string | 是 | - | OBS
文件系统的访问密钥 |
| endpoint | string | 是 | - | OBS 文件系统的端点
|
| read_columns | list | 否 | - | 数据源的读取列列表
|
+| sheet_name | string | 否 | - |
读取工作簿的工作表，仅在 file_format 为 excel 时使用。
|
+| excel_engine | string | 否 | POI | 仅在
`file_format` 为 excel 时使用。支持的引擎包括 `POI` 和 `EasyExcel`。
|
+| poi_excel_max_file_size | long | 否 | 52428800 | 仅在
`file_format` 为 excel 且 `excel_engine` 为 POI 时使用。POI 引擎允许读取的最大 Excel 文件大小（默认 50
MB）。
|
| delimiter | string | 否 | \001 | 字段分隔符
|
| row_delimiter | string | 否 | \n | 行分隔符
|
| parse_partition_from_path | boolean | 否 | true |
控制是否从文件路径解析分区键和值 |
diff --git a/docs/zh/connectors/source/OssFile.md
b/docs/zh/connectors/source/OssFile.md
index 510458b0d7..4572d66edb 100644
--- a/docs/zh/connectors/source/OssFile.md
+++ b/docs/zh/connectors/source/OssFile.md
@@ -204,6 +204,8 @@ schema {
| csv_use_header_line | boolean | 否 | false |
是否使用标题行来解析文件，仅在file_format为`csv`且文件包含符合RFC 4180的标题行时使用
|
| schema | config | 否 | - |
上游数据的schema。
|
| sheet_name | string | 否 | - |
读取工作簿的工作表，仅在file_format为excel时使用。
|
+| excel_engine | string | 否 | POI | 仅在
`file_format` 为 excel 时使用。支持的引擎包括 `POI` 和 `EasyExcel`。
|
+| poi_excel_max_file_size | long | 否 | 52428800 | 仅在
`file_format` 为 excel 且 `excel_engine` 为 POI 时使用。POI 引擎允许读取的最大 Excel 文件大小（默认 50
MB）。
|
| xml_row_tag | string | 否 | - |
指定XML文件中数据行的标签名称，仅在file_format为xml时使用。
|
| xml_use_attr_format | boolean | 否 | - |
指定是否使用标签属性格式处理数据，仅在file_format为xml时使用。
|
| compress_codec | string | 否 | none |
文件使用的压缩编解码器。
|
diff --git a/docs/zh/connectors/source/OssJindoFile.md
b/docs/zh/connectors/source/OssJindoFile.md
index 81fbcf24ac..f6763cc518 100644
--- a/docs/zh/connectors/source/OssJindoFile.md
+++ b/docs/zh/connectors/source/OssJindoFile.md
@@ -75,6 +75,8 @@ import ChangeLog from
'../changelog/connector-file-oss-jindo.md';
| skip_header_row_number | long | 否 | 0 |
跳过前几行 |
| schema | config | 否 | - |
上游数据的模式信息。更多详情请参考 [Schema 特性](../../introduction/concepts/schema-feature.md)。 |
| sheet_name | string | 否 | - |
Excel 工作表名称 |
+| excel_engine | string | 否 | POI | 仅在
`file_format` 为 excel 时使用。支持的引擎包括 `POI` 和 `EasyExcel`。
|
+| poi_excel_max_file_size | long | 否 | 52428800 | 仅在
`file_format` 为 excel 且 `excel_engine` 为 POI 时使用。POI 引擎允许读取的最大 Excel 文件大小（默认 50
MB）。
|
| xml_row_tag | string | 否 | - | XML
行标签 |
| xml_use_attr_format | boolean | 否 | - |
是否使用 XML 属性格式 |
| csv_use_header_line | boolean | 否 | false |
是否使用 CSV 标题行 |
diff --git a/docs/zh/connectors/source/S3File.md
b/docs/zh/connectors/source/S3File.md
index aba0625d58..9fbb78d989 100644
--- a/docs/zh/connectors/source/S3File.md
+++ b/docs/zh/connectors/source/S3File.md
@@ -211,6 +211,8 @@ schema {
| csv_use_header_line | boolean | 否 | false
| 是否使用标题行来解析文件，仅在file_format为`csv`且文件包含符合RFC
4180的标题行时使用
|
| schema | config | 否 | -
| 上游数据的schema。更多详情请参考 [Schema
特性](../../introduction/concepts/schema-feature.md)。
|
| sheet_name | string | 否 | -
| 读取工作簿的工作表，仅在file_format为excel时使用。
|
+| excel_engine | string | 否 | POI
| 仅在 `file_format` 为 excel 时使用。支持的引擎包括 `POI` 和
`EasyExcel`。
|
+| poi_excel_max_file_size | long | 否 | 52428800
| 仅在 `file_format` 为 excel 且 `excel_engine` 为 POI
时使用。POI 引擎允许读取的最大 Excel 文件大小（默认 50 MB）。
|
| xml_row_tag | string | 否 | -
| 指定XML文件中数据行的标签名称，仅对XML文件有效。
|
| xml_use_attr_format | boolean | 否 | -
| 指定是否使用标签属性格式处理数据，仅对XML文件有效。
|
| compress_codec | string | 否 | none
|
|
diff --git a/docs/zh/connectors/source/SftpFile.md
b/docs/zh/connectors/source/SftpFile.md
index 2e8c123a58..f8ec822651 100644
--- a/docs/zh/connectors/source/SftpFile.md
+++ b/docs/zh/connectors/source/SftpFile.md
@@ -99,6 +99,8 @@ import ChangeLog from '../changelog/connector-file-sftp.md';
| skip_header_row_number | Long | 否 | 0 |
跳过前几行，但仅适用于txt和csv。<br/> 例如，设置如下：<br/> `skip_header_row_number = 2` <br/>
然后SeaTunnel将跳过源文件的前2行
|
| read_columns | list | 否 | - |
数据源的读取列列表，用户可以使用它来实现字段投影。
|
| sheet_name | String | 否 | - |
读取工作簿的工作表，仅在file_format为excel时使用。
|
+| excel_engine | string | 否 | POI | 仅在
`file_format` 为 excel 时使用。支持的引擎包括 `POI` 和 `EasyExcel`。
|
+| poi_excel_max_file_size | long | 否 | 52428800 | 仅在
`file_format` 为 excel 且 `excel_engine` 为 POI 时使用。POI 引擎允许读取的最大 Excel 文件大小（默认 50
MB）。
|
| xml_row_tag | string | 否 | - |
指定XML文件中数据行的标签名称，仅在file_format为xml时使用。
|
| xml_use_attr_format | boolean | 否 | - |
指定是否使用标签属性格式处理数据，仅在file_format为xml时使用。
|
| csv_use_header_line | boolean | 否 | false |
是否使用标题行来解析文件，仅在file_format为`csv`且文件包含符合RFC 4180的标题行时使用
|
diff --git a/docs/zh/introduction/concepts/incompatible-changes.md
b/docs/zh/introduction/concepts/incompatible-changes.md
index caddbf3172..a9b52589db 100644
--- a/docs/zh/introduction/concepts/incompatible-changes.md
+++ b/docs/zh/introduction/concepts/incompatible-changes.md
@@ -109,6 +109,12 @@
- **已存在的 Iceberg 表**（Glue/Hive 元数据中已有 `identifier-field-ids`）在运行时不受影响；
只有 sink 新建的表会改变行为。
+- **破坏性变更：File 源连接器拒绝大于 `poi_excel_max_file_size`（默认 50 MB）的 POI 引擎 Excel 文件**
+ -
**影响范围**：`seatunnel-connectors-v2/connector-file`（LocalFile、HdfsFile、S3File、FtpFile、SftpFile、OssFile、OssJindoFile、ObsFile、CosFile）
+ - **变更说明**：Apache POI 在读取任何行之前会将整个 Excel 工作簿完全加载到内存，对于较大的 `.xls`/`.xlsx`
文件可能导致 Zeta worker 严重 GC 压力甚至 OOM。新增 `poi_excel_max_file_size` 选项（默认 50 MB），POI
在构建工作簿之前会拒绝超过该限制的 Excel 文件。该校验同时覆盖普通 Excel 文件和归档（ZIP/TAR/TAR_GZ/GZ）中的 Excel
条目，且仅在 `excel_engine = POI`（默认值）时生效；流式读取的 `excel_engine = EasyExcel` 路径不受此限制。
+ - **影响**：此前以 POI 引擎读取大于 50 MB Excel 文件的任务（虽然成功但伴随严重内存压力）现在会以
`FileConnectorException` 快速失败，而不再可能导致 worker OOM。
+ - **迁移指南**：对于必须读取大 Excel 文件且 worker 内存充足的 POI 任务，可通过
`poi_excel_max_file_size = <字节数>` 调高限制；否则切换为 `excel_engine =
EasyExcel`，该引擎惰性流式读取行，不受此限制约束。
+
### 转换变更
- **[BREAKING]** SQL Transform 的 `PARSEDATETIME`、`TO_DATE` 和 `IS_DATE`
函数现在只接受白名单中的日期时间格式模式。以前接受的自定义格式模式现在将在运行时失败。支持的模式有：
diff --git
a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/FileBaseOptions.java
b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/FileBaseOptions.java
index 5f078f59b4..a7cd9ecf6e 100644
---
a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/FileBaseOptions.java
+++
b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/FileBaseOptions.java
@@ -26,6 +26,7 @@ import java.util.List;
import static
org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
public class FileBaseOptions extends ConnectorCommonOptions {
+ public static final long DEFAULT_POI_EXCEL_MAX_FILE_SIZE = 50L * 1024L *
1024L;
public static final Option<String> FILENAME_EXTENSION =
Options.key("filename_extension")
@@ -118,7 +119,14 @@ public class FileBaseOptions extends
ConnectorCommonOptions {
Options.key("excel_engine")
.enumType(ExcelEngine.class)
.defaultValue(ExcelEngine.POI)
- .withDescription("To switch excel read engine, e.g. POI ,
EasyExcel");
+ .withDescription("To switch excel read engine, e.g. POI,
EasyExcel");
+
+ public static final Option<Long> POI_EXCEL_MAX_FILE_SIZE =
+ Options.key("poi_excel_max_file_size")
+ .longType()
+ .defaultValue(DEFAULT_POI_EXCEL_MAX_FILE_SIZE)
+ .withDescription(
+ "Maximum Excel file size in bytes allowed by POI
engine. Use EasyExcel for larger Excel files.");
public static final Option<String> XML_ROW_TAG =
Options.key("xml_row_tag")
diff --git
a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/FileBaseSourceOptions.java
b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/FileBaseSourceOptions.java
index ec8f2f75c7..edffd618d8 100644
---
a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/FileBaseSourceOptions.java
+++
b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/FileBaseSourceOptions.java
@@ -28,6 +28,7 @@ import java.util.Map;
public class FileBaseSourceOptions extends FileBaseOptions {
public static final String DEFAULT_ROW_DELIMITER = "\n";
+ public static final long DEFAULT_POI_EXCEL_MAX_FILE_SIZE = 50L * 1024L *
1024L;
public static final Option<FileDiscoveryMode> DISCOVERY_MODE =
Options.key("discovery_mode")
@@ -128,7 +129,14 @@ public class FileBaseSourceOptions extends FileBaseOptions
{
Options.key("excel_engine")
.enumType(ExcelEngine.class)
.defaultValue(ExcelEngine.POI)
- .withDescription("To switch excel read engine, e.g. POI ,
EasyExcel");
+ .withDescription("To switch excel read engine, e.g. POI,
EasyExcel");
+
+ public static final Option<Long> POI_EXCEL_MAX_FILE_SIZE =
+ Options.key("poi_excel_max_file_size")
+ .longType()
+ .defaultValue(DEFAULT_POI_EXCEL_MAX_FILE_SIZE)
+ .withDescription(
+ "Maximum Excel file size in bytes allowed by POI
engine. Use EasyExcel for larger Excel files.");
public static final Option<String> XML_ROW_TAG =
Options.key("xml_row_tag")
diff --git
a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategy.java
b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategy.java
index 99dd5fceed..ac998f60c3 100644
---
a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategy.java
+++
b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategy.java
@@ -30,6 +30,7 @@ import org.apache.seatunnel.api.table.type.BasicType;
import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
+import org.apache.seatunnel.common.exception.CommonErrorCodeDeprecated;
import org.apache.seatunnel.common.utils.SeaTunnelException;
import
org.apache.seatunnel.connectors.seatunnel.file.config.ArchiveCompressFormat;
import
org.apache.seatunnel.connectors.seatunnel.file.config.FileBaseSourceOptions;
@@ -428,8 +429,20 @@ public abstract class AbstractReadStrategy implements
ReadStrategy {
Map<String, String> partitionsMap,
FileFormat fileFormat)
throws IOException {
+ resolveArchiveCompressedInputStream(
+ split, output, partitionsMap, fileFormat, Long.valueOf(-1L));
+ }
+
+ protected void resolveArchiveCompressedInputStream(
+ FileSourceSplit split,
+ Collector<SeaTunnelRow> output,
+ Map<String, String> partitionsMap,
+ FileFormat fileFormat,
+ Long maxBytesForEntry)
+ throws IOException {
String path = split.getFilePath();
String tableId = split.getTableId();
+ long effectiveMaxBytes = maxBytesForEntry == null ? -1L :
maxBytesForEntry;
switch (archiveCompressFormat) {
case ZIP:
try (ZipInputStream zis =
@@ -437,10 +450,12 @@ public abstract class AbstractReadStrategy implements
ReadStrategy {
ZipEntry entry;
while ((entry = zis.getNextEntry()) != null) {
if (!entry.isDirectory() &&
checkFileType(entry.getName(), fileFormat)) {
+ assertArchiveEntrySize(
+ entry.getName(), entry.getSize(),
effectiveMaxBytes);
readProcess(
split,
output,
- copyInputStream(zis),
+ copyInputStream(zis, effectiveMaxBytes),
partitionsMap,
entry.getName());
}
@@ -454,10 +469,12 @@ public abstract class AbstractReadStrategy implements
ReadStrategy {
TarArchiveEntry entry;
while ((entry = tarInput.getNextTarEntry()) != null) {
if (!entry.isDirectory() &&
checkFileType(entry.getName(), fileFormat)) {
+ assertArchiveEntrySize(
+ entry.getName(), entry.getSize(),
effectiveMaxBytes);
readProcess(
split,
output,
- copyInputStream(tarInput),
+ copyInputStream(tarInput,
effectiveMaxBytes),
partitionsMap,
entry.getName());
}
@@ -473,10 +490,12 @@ public abstract class AbstractReadStrategy implements
ReadStrategy {
TarArchiveEntry entry;
while ((entry = tarIn.getNextTarEntry()) != null) {
if (!entry.isDirectory() &&
checkFileType(entry.getName(), fileFormat)) {
+ assertArchiveEntrySize(
+ entry.getName(), entry.getSize(),
effectiveMaxBytes);
readProcess(
split,
output,
- copyInputStream(tarIn),
+ copyInputStream(tarIn, effectiveMaxBytes),
partitionsMap,
entry.getName());
}
@@ -502,7 +521,12 @@ public abstract class AbstractReadStrategy implements
ReadStrategy {
fileName = path;
}
}
- readProcess(split, output, copyInputStream(gzipIn),
partitionsMap, fileName);
+ readProcess(
+ split,
+ output,
+ copyInputStream(gzipIn, effectiveMaxBytes),
+ partitionsMap,
+ fileName);
}
break;
case NONE:
@@ -526,6 +550,25 @@ public abstract class AbstractReadStrategy implements
ReadStrategy {
}
}
+ /**
+ * Rejects an archive entry whose declared size exceeds the configured POI
limit.
+ *
+ * @param entryName archive entry name used in the error message
+ * @param entrySize declared uncompressed entry size in bytes
+ * @param maxBytes maximum allowed size in bytes; non-positive values
disable the limit
+ */
+ private void assertArchiveEntrySize(String entryName, long entrySize, long
maxBytes) {
+ if (maxBytes <= 0 || entrySize <= 0 || entrySize <= maxBytes) {
+ return;
+ }
+ throw new FileConnectorException(
+ CommonErrorCodeDeprecated.UNSUPPORTED_OPERATION,
+ String.format(
+ "Archived entry [%s] is %,d bytes, larger than POI
limit %,d bytes. "
+ + "Please set excel_engine = EasyExcel, or
increase the limit if POI is required.",
+ entryName, entrySize, maxBytes));
+ }
+
protected void readProcess(
FileSourceSplit split,
Collector<SeaTunnelRow> output,
@@ -608,11 +651,35 @@ public abstract class AbstractReadStrategy implements
ReadStrategy {
}
protected static InputStream copyInputStream(InputStream inputStream)
throws IOException {
+ return copyInputStream(inputStream, -1L);
+ }
+
+ /**
+ * Copies an input stream into memory, optionally stopping once it exceeds
a byte limit.
+ *
+ * @param inputStream source stream
+ * @param maxBytes maximum number of bytes to copy; non-positive values
disable the limit
+ * @return an input stream backed by the copied bytes
+ * @throws IOException if reading from the source stream fails
+ * @throws FileConnectorException if the copied data exceeds {@code
maxBytes}
+ */
+ protected static InputStream copyInputStream(InputStream inputStream, long
maxBytes)
+ throws IOException {
ByteArrayOutputStream byteArrayOutputStream = new
ByteArrayOutputStream();
byte[] buffer = new byte[1024];
int bytesRead;
+ long total = 0;
while ((bytesRead = inputStream.read(buffer)) != -1) {
+ total += bytesRead;
+ if (maxBytes > 0 && total > maxBytes) {
+ throw new FileConnectorException(
+ CommonErrorCodeDeprecated.UNSUPPORTED_OPERATION,
+ String.format(
+ "Archived entry exceeds %,d bytes (POI limit).
"
+ + "Please set excel_engine =
EasyExcel, or increase the limit if POI is required.",
+ maxBytes));
+ }
byteArrayOutputStream.write(buffer, 0, bytesRead);
}
diff --git
a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/ExcelReadStrategy.java
b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/ExcelReadStrategy.java
index 145a551de5..d63143f278 100644
---
a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/ExcelReadStrategy.java
+++
b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/ExcelReadStrategy.java
@@ -26,6 +26,7 @@ import
org.apache.seatunnel.common.exception.CommonErrorCodeDeprecated;
import org.apache.seatunnel.common.utils.DateTimeUtils;
import org.apache.seatunnel.common.utils.DateUtils;
import org.apache.seatunnel.common.utils.TimeUtils;
+import
org.apache.seatunnel.connectors.seatunnel.file.config.ArchiveCompressFormat;
import org.apache.seatunnel.connectors.seatunnel.file.config.ExcelEngine;
import
org.apache.seatunnel.connectors.seatunnel.file.config.FileBaseSourceOptions;
import org.apache.seatunnel.connectors.seatunnel.file.config.FileFormat;
@@ -78,8 +79,17 @@ public class ExcelReadStrategy extends AbstractReadStrategy {
@Override
public void read(String path, String tableId, Collector<SeaTunnelRow>
output) {
Map<String, String> partitionsMap = parsePartitionsByPath(path);
+ // Only enforce the POI file-size guard when the user explicitly chose
the POI
+ // engine. EasyExcel streams rows lazily, so the same bound should not
apply —
+ // otherwise the documented escape hatch would silently break for
archived files.
+ long maxBytesForEntry =
+ ExcelEngine.EASY_EXCEL.equals(getExcelEngine()) ? -1L :
getPoiExcelMaxFileSize();
resolveArchiveCompressedInputStream(
- new FileSourceSplit(tableId, path), output, partitionsMap,
FileFormat.EXCEL);
+ new FileSourceSplit(tableId, path),
+ output,
+ partitionsMap,
+ FileFormat.EXCEL,
+ maxBytesForEntry);
}
@Override
@@ -117,97 +127,222 @@ public class ExcelReadStrategy extends
AbstractReadStrategy {
dateTimeFormatterPattern,
timeFormatterPattern);
- if (pluginConfig.hasPath(FileBaseSourceOptions.EXCEL_ENGINE.key())
- && pluginConfig
- .getString(FileBaseSourceOptions.EXCEL_ENGINE.key())
- .equals(ExcelEngine.EASY_EXCEL.getExcelEngineName())) {
- log.info("Parsing Excel with EasyExcel");
+ ExcelEngine excelEngine = getExcelEngine();
+ if (ExcelEngine.EASY_EXCEL.equals(excelEngine)) {
+ readByEasyExcel(tableId, output, inputStream, excelCellUtils);
+ } else {
+ readByPoi(
+ split,
+ tableId,
+ output,
+ inputStream,
+ partitionsMap,
+ currentFileName,
+ excelCellUtils);
+ }
+ }
- ExcelReaderBuilder read =
- EasyExcel.read(
- inputStream,
- new ExcelReaderListener(
- tableId, output, excelCellUtils,
seaTunnelRowType));
- if (pluginConfig.hasPath(FileBaseSourceOptions.SHEET_NAME.key())) {
-
read.sheet(pluginConfig.getString(FileBaseSourceOptions.SHEET_NAME.key()))
- .headRowNumber((int) skipHeaderNumber)
- .doReadSync();
- } else {
- read.sheet(0).headRowNumber((int)
skipHeaderNumber).doReadSync();
+ /**
+ * Resolves the configured Excel engine, using the default when no engine
is configured.
+ *
+ * @return the configured or default Excel engine
+ * @throws FileConnectorException if the configured engine is unsupported
+ */
+ private ExcelEngine getExcelEngine() {
+ if (!pluginConfig.hasPath(FileBaseSourceOptions.EXCEL_ENGINE.key())) {
+ return FileBaseSourceOptions.EXCEL_ENGINE.defaultValue();
+ }
+ String configuredExcelEngine =
+
pluginConfig.getString(FileBaseSourceOptions.EXCEL_ENGINE.key());
+ for (ExcelEngine excelEngine : ExcelEngine.values()) {
+ if (excelEngine.name().equalsIgnoreCase(configuredExcelEngine)
+ ||
excelEngine.getExcelEngineName().equalsIgnoreCase(configuredExcelEngine)) {
+ return excelEngine;
}
+ }
+ throw new FileConnectorException(
+ CommonErrorCodeDeprecated.ILLEGAL_ARGUMENT,
+ "Unsupported excel_engine: " + configuredExcelEngine);
+ }
+
+ /**
+ * Reads an Excel stream with EasyExcel without applying the POI file-size
limit.
+ *
+ * @param tableId table identifier
+ * @param output row collector
+ * @param inputStream Excel input stream
+ * @param excelCellUtils cell conversion utility
+ */
+ private void readByEasyExcel(
+ String tableId,
+ Collector<SeaTunnelRow> output,
+ InputStream inputStream,
+ ExcelCellUtils excelCellUtils) {
+ log.info("Parsing Excel with EasyExcel");
+
+ ExcelReaderBuilder read =
+ EasyExcel.read(
+ inputStream,
+ new ExcelReaderListener(tableId, output,
excelCellUtils, seaTunnelRowType));
+ if (pluginConfig.hasPath(FileBaseSourceOptions.SHEET_NAME.key())) {
+
read.sheet(pluginConfig.getString(FileBaseSourceOptions.SHEET_NAME.key()))
+ .headRowNumber((int) skipHeaderNumber)
+ .doReadSync();
} else {
- log.info("Parsing Excel with POI");
+ read.sheet(0).headRowNumber((int) skipHeaderNumber).doReadSync();
+ }
+ }
- Workbook workbook;
- FormulaEvaluator formulaEvaluator;
- if (currentFileName.endsWith(".xls")) {
- workbook = new HSSFWorkbook(inputStream);
- formulaEvaluator =
workbook.getCreationHelper().createFormulaEvaluator();
- } else if (currentFileName.endsWith(".xlsx")) {
- workbook = new XSSFWorkbook(inputStream);
- formulaEvaluator = new XSSFFormulaEvaluator((XSSFWorkbook)
workbook);
- } else {
- throw new FileConnectorException(
- CommonErrorCodeDeprecated.UNSUPPORTED_OPERATION,
- "Only support read excel file");
- }
- DataFormatter formatter = new DataFormatter();
- Sheet sheet =
-
pluginConfig.hasPath(FileBaseSourceOptions.SHEET_NAME.key())
- ? workbook.getSheet(
-
pluginConfig.getString(FileBaseSourceOptions.SHEET_NAME.key()))
- : workbook.getSheetAt(0);
- cellCount = seaTunnelRowType.getTotalFields();
- cellCount = partitionsMap.isEmpty() ? cellCount : cellCount +
partitionsMap.size();
- SeaTunnelDataType<?>[] fieldTypes =
seaTunnelRowType.getFieldTypes();
- int firstRowNum = sheet.getFirstRowNum();
- int lastRowNum = sheet.getLastRowNum();
- if (firstRowNum == -1 || lastRowNum == -1) {
- return;
- }
- // Calculate the actual start row considering skipHeaderNumber
- int startRow = Math.max(firstRowNum + (int) skipHeaderNumber,
firstRowNum);
- if (startRow > lastRowNum) {
- throw new FileConnectorException(
- CommonErrorCodeDeprecated.UNSUPPORTED_OPERATION,
- "Skip the number of rows exceeds the maximum or
minimum limit of Sheet");
- }
- IntStream.range(startRow, lastRowNum + 1)
- .mapToObj(sheet::getRow)
- .filter(Objects::nonNull)
- .forEach(
- rowData -> {
- int[] cellIndexes =
- indexes == null
- ? IntStream.range(0,
cellCount).toArray()
- : indexes;
- int z = 0;
- SeaTunnelRow seaTunnelRow = new
SeaTunnelRow(cellCount);
- for (int j : cellIndexes) {
- Cell cell = rowData.getCell(j);
- seaTunnelRow.setField(
- z++,
- cell == null
- ? null
- : excelCellUtils.convert(
- getCellValue(
-
cell.getCellType(),
- cell,
-
formulaEvaluator,
- formatter),
- fieldTypes[z - 1],
- null));
- }
- if (isMergePartition) {
- int index =
seaTunnelRowType.getTotalFields();
- for (String value :
partitionsMap.values()) {
- seaTunnelRow.setField(index++, value);
- }
+ /**
+ * Reads an Excel stream with Apache POI after guarding direct files
against oversized
+ * workbooks. Archived entries are guarded before this method is called.
+ *
+ * @param split source split for the current file
+ * @param tableId table identifier
+ * @param output row collector
+ * @param inputStream Excel input stream
+ * @param partitionsMap partition values inferred from the path
+ * @param currentFileName file name used to select the workbook type
+ * @param excelCellUtils cell conversion utility
+ * @throws IOException if the file size or workbook cannot be read
+ */
+ private void readByPoi(
+ FileSourceSplit split,
+ String tableId,
+ Collector<SeaTunnelRow> output,
+ InputStream inputStream,
+ Map<String, String> partitionsMap,
+ String currentFileName,
+ ExcelCellUtils excelCellUtils)
+ throws IOException {
+ // For archived reads (ZIP/TAR/TAR_GZ/GZ) the entry-level guard
+ // (assertArchiveEntrySize + bounded copy) already rejected oversized
+ // entries. The split here carries the outer archive path rather than
the
+ // individual entry, so re-statting it would measure the archive's
on-disk
+ // size and falsely reject a small Excel entry bundled inside a larger
+ // archive. Only enforce the POI-level guard for the non-archived
(direct
+ // file) path, where the split path is the Excel file itself.
+ if (archiveCompressFormat == ArchiveCompressFormat.NONE) {
+ assertPoiFileSize(split, currentFileName);
+ }
+ log.info("Parsing Excel with POI");
+
+ Workbook workbook;
+ FormulaEvaluator formulaEvaluator;
+ if (currentFileName.endsWith(".xls")) {
+ workbook = new HSSFWorkbook(inputStream);
+ formulaEvaluator =
workbook.getCreationHelper().createFormulaEvaluator();
+ } else if (currentFileName.endsWith(".xlsx")) {
+ workbook = new XSSFWorkbook(inputStream);
+ formulaEvaluator = new XSSFFormulaEvaluator((XSSFWorkbook)
workbook);
+ } else {
+ throw new FileConnectorException(
+ CommonErrorCodeDeprecated.UNSUPPORTED_OPERATION,
+ "Only support read excel file");
+ }
+ DataFormatter formatter = new DataFormatter();
+ Sheet sheet =
+ pluginConfig.hasPath(FileBaseSourceOptions.SHEET_NAME.key())
+ ? workbook.getSheet(
+
pluginConfig.getString(FileBaseSourceOptions.SHEET_NAME.key()))
+ : workbook.getSheetAt(0);
+ cellCount = seaTunnelRowType.getTotalFields();
+ cellCount = partitionsMap.isEmpty() ? cellCount : cellCount +
partitionsMap.size();
+ SeaTunnelDataType<?>[] fieldTypes = seaTunnelRowType.getFieldTypes();
+ int firstRowNum = sheet.getFirstRowNum();
+ int lastRowNum = sheet.getLastRowNum();
+ if (firstRowNum == -1 || lastRowNum == -1) {
+ return;
+ }
+ // Calculate the actual start row considering skipHeaderNumber
+ int startRow = Math.max(firstRowNum + (int) skipHeaderNumber,
firstRowNum);
+ if (startRow > lastRowNum) {
+ throw new FileConnectorException(
+ CommonErrorCodeDeprecated.UNSUPPORTED_OPERATION,
+ "Skip the number of rows exceeds the maximum or minimum
limit of Sheet");
+ }
+ IntStream.range(startRow, lastRowNum + 1)
+ .mapToObj(sheet::getRow)
+ .filter(Objects::nonNull)
+ .forEach(
+ rowData -> {
+ int[] cellIndexes =
+ indexes == null
+ ? IntStream.range(0,
cellCount).toArray()
+ : indexes;
+ int z = 0;
+ SeaTunnelRow seaTunnelRow = new
SeaTunnelRow(cellCount);
+ for (int j : cellIndexes) {
+ Cell cell = rowData.getCell(j);
+ seaTunnelRow.setField(
+ z++,
+ cell == null
+ ? null
+ : excelCellUtils.convert(
+ getCellValue(
+
cell.getCellType(),
+ cell,
+
formulaEvaluator,
+ formatter),
+ fieldTypes[z - 1],
+ null));
+ }
+ if (isMergePartition) {
+ int index = seaTunnelRowType.getTotalFields();
+ for (String value : partitionsMap.values()) {
+ seaTunnelRow.setField(index++, value);
}
- seaTunnelRow.setTableId(tableId);
- output.collect(seaTunnelRow);
- });
+ }
+ seaTunnelRow.setTableId(tableId);
+ output.collect(seaTunnelRow);
+ });
+ }
+
+ /**
+ * Checks a direct Excel file's size before POI materializes the workbook
in memory.
+ *
+ * @param split source split for the current file
+ * @param currentFileName file name used in the error message
+ * @throws IOException if the file status cannot be read
+ */
+ private void assertPoiFileSize(FileSourceSplit split, String
currentFileName)
+ throws IOException {
+ long maxFileSize = getPoiExcelMaxFileSize();
+ long fileSize =
+ split.getLength() > -1
+ ? split.getLength()
+ :
hadoopFileSystemProxy.getFileStatus(split.getFilePath()).getLen();
+ if (fileSize > maxFileSize) {
+ throw new FileConnectorException(
+ CommonErrorCodeDeprecated.UNSUPPORTED_OPERATION,
+ String.format(
+ "Excel file [%s] is %,d bytes, larger than POI
limit %,d bytes. "
+ + "Please set excel_engine = EasyExcel, or
increase %s if POI is required.",
+ currentFileName,
+ fileSize,
+ maxFileSize,
+
FileBaseSourceOptions.POI_EXCEL_MAX_FILE_SIZE.key()));
+ }
+ }
+
+ /**
+ * Resolves and validates the maximum file size allowed for POI Excel
reads.
+ *
+ * @return the maximum allowed size in bytes
+ * @throws FileConnectorException if the configured limit is not positive
+ */
+ private long getPoiExcelMaxFileSize() {
+ long maxFileSize =
+
pluginConfig.hasPath(FileBaseSourceOptions.POI_EXCEL_MAX_FILE_SIZE.key())
+ ?
pluginConfig.getLong(FileBaseSourceOptions.POI_EXCEL_MAX_FILE_SIZE.key())
+ :
FileBaseSourceOptions.POI_EXCEL_MAX_FILE_SIZE.defaultValue();
+ if (maxFileSize <= 0) {
+ throw new FileConnectorException(
+ CommonErrorCodeDeprecated.ILLEGAL_ARGUMENT,
+ FileBaseSourceOptions.POI_EXCEL_MAX_FILE_SIZE.key()
+ + " must be greater than 0");
}
+ return maxFileSize;
}
@Override
diff --git
a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/reader/ExcelReadStrategyTest.java
b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/reader/ExcelReadStrategyTest.java
index 91b4d7041d..3186f3421d 100644
---
a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/reader/ExcelReadStrategyTest.java
+++
b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/reader/ExcelReadStrategyTest.java
@@ -28,6 +28,7 @@ import org.apache.seatunnel.common.utils.DateTimeUtils;
import org.apache.seatunnel.common.utils.DateUtils;
import org.apache.seatunnel.common.utils.TimeUtils;
import org.apache.seatunnel.connectors.seatunnel.file.config.HadoopConf;
+import
org.apache.seatunnel.connectors.seatunnel.file.exception.FileConnectorException;
import
org.apache.seatunnel.connectors.seatunnel.file.source.reader.ExcelReadStrategy;
import org.junit.jupiter.api.Assertions;
@@ -36,10 +37,12 @@ import org.junit.jupiter.api.Test;
import lombok.Getter;
import java.io.File;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.math.BigDecimal;
import java.net.URISyntaxException;
import java.net.URL;
+import java.nio.file.Files;
import java.nio.file.Paths;
import java.time.LocalDate;
import java.time.LocalDateTime;
@@ -47,6 +50,9 @@ import java.time.LocalTime;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
+import java.util.zip.CRC32;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
import static
org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_DEFAULT;
@@ -237,6 +243,204 @@ public class ExcelReadStrategyTest {
testLargeExcelRead("/excel/e2e.xlsx", "/excel/e2exls.conf", 5);
}
+ @Test
+ public void testEasyExcelIgnoresPoiFileSizeLimit() throws IOException,
URISyntaxException {
+ URL excelFile =
ExcelReadStrategyTest.class.getResource("/excel/e2e.xlsx");
+ URL conf =
ExcelReadStrategyTest.class.getResource("/excel/e2exls.conf");
+
+ Assertions.assertNotNull(excelFile);
+ Assertions.assertNotNull(conf);
+ String excelFilePath = Paths.get(excelFile.toURI()).toString();
+ String confPath = Paths.get(conf.toURI()).toString();
+ Config pluginConfig =
+ ConfigFactory.parseString("poi_excel_max_file_size = 1")
+ .withFallback(ConfigFactory.parseFile(new
File(confPath)));
+ ExcelReadStrategy excelReadStrategy = new ExcelReadStrategy();
+ LocalConf localConf = new LocalConf(FS_DEFAULT_NAME_DEFAULT);
+ excelReadStrategy.setPluginConfig(pluginConfig);
+ excelReadStrategy.init(localConf);
+
+ List<String> fileNamesByPath =
excelReadStrategy.getFileNamesByPath(excelFilePath);
+ CatalogTable userDefinedCatalogTable =
CatalogTableUtil.buildWithConfig(pluginConfig);
+ excelReadStrategy.setCatalogTable(userDefinedCatalogTable);
+
+ TestCollector testCollector = new TestCollector();
+ excelReadStrategy.read(fileNamesByPath.get(0), "", testCollector);
+
+ Assertions.assertEquals(5, testCollector.getRows().size());
+ }
+
+ @Test
+ public void testPoiRejectsExcelLargerThanConfiguredLimit()
+ throws IOException, URISyntaxException {
+ URL excelFile =
ExcelReadStrategyTest.class.getResource("/excel/e2e.xlsx");
+ URL conf =
ExcelReadStrategyTest.class.getResource("/excel/e2exls.conf");
+
+ Assertions.assertNotNull(excelFile);
+ Assertions.assertNotNull(conf);
+ String excelFilePath = Paths.get(excelFile.toURI()).toString();
+ String confPath = Paths.get(conf.toURI()).toString();
+ Config pluginConfig =
+ ConfigFactory.parseString("poi_excel_max_file_size = 1")
+ .withFallback(ConfigFactory.parseFile(new
File(confPath)))
+ .withoutPath("excel_engine");
+ ExcelReadStrategy excelReadStrategy = new ExcelReadStrategy();
+ LocalConf localConf = new LocalConf(FS_DEFAULT_NAME_DEFAULT);
+ excelReadStrategy.setPluginConfig(pluginConfig);
+ excelReadStrategy.init(localConf);
+
+ List<String> fileNamesByPath =
excelReadStrategy.getFileNamesByPath(excelFilePath);
+ CatalogTable userDefinedCatalogTable =
CatalogTableUtil.buildWithConfig(pluginConfig);
+ excelReadStrategy.setCatalogTable(userDefinedCatalogTable);
+
+ FileConnectorException exception =
+ Assertions.assertThrows(
+ FileConnectorException.class,
+ () ->
+ excelReadStrategy.read(
+ fileNamesByPath.get(0), "", new
TestCollector()));
+ Assertions.assertTrue(exception.getMessage().contains("larger than POI
limit"));
+ Assertions.assertTrue(exception.getMessage().contains("excel_engine =
EasyExcel"));
+ }
+
+ @Test
+ public void testPoiRejectsArchivedExcelEntryLargerThanConfiguredLimit()
+ throws IOException, URISyntaxException {
+ URL zipFile =
ExcelReadStrategyTest.class.getResource("/excel/archive_zip/e2e_in_zip.zip");
+ URL conf =
ExcelReadStrategyTest.class.getResource("/excel/e2exls.conf");
+
+ Assertions.assertNotNull(zipFile);
+ Assertions.assertNotNull(conf);
+ String zipFilePath = Paths.get(zipFile.toURI()).toString();
+ String confPath = Paths.get(conf.toURI()).toString();
+ Config pluginConfig =
+ ConfigFactory.parseString(
+ "poi_excel_max_file_size =
1\narchive_compress_codec = \"ZIP\"")
+ .withFallback(ConfigFactory.parseFile(new
File(confPath)))
+ .withoutPath("excel_engine");
+ ExcelReadStrategy excelReadStrategy = new ExcelReadStrategy();
+ LocalConf localConf = new LocalConf(FS_DEFAULT_NAME_DEFAULT);
+ excelReadStrategy.setPluginConfig(pluginConfig);
+ excelReadStrategy.init(localConf);
+
+ List<String> fileNamesByPath =
excelReadStrategy.getFileNamesByPath(zipFilePath);
+ CatalogTable userDefinedCatalogTable =
CatalogTableUtil.buildWithConfig(pluginConfig);
+ excelReadStrategy.setCatalogTable(userDefinedCatalogTable);
+
+ FileConnectorException exception =
+ Assertions.assertThrows(
+ FileConnectorException.class,
+ () ->
+ excelReadStrategy.read(
+ fileNamesByPath.get(0), "", new
TestCollector()));
+ Assertions.assertTrue(
+ exception.getMessage().contains("larger than POI limit")
+ || exception.getMessage().contains("is %,d bytes,
larger than POI limit"),
+ "Expected archived entry guard error, got: " +
exception.getMessage());
+ }
+
+ @Test
+ public void testEasyExcelReadsArchivedExcelEntryAbovePoiLimit()
+ throws IOException, URISyntaxException {
+ URL zipFile =
ExcelReadStrategyTest.class.getResource("/excel/archive_zip/e2e_in_zip.zip");
+ URL conf =
ExcelReadStrategyTest.class.getResource("/excel/e2exls.conf");
+
+ Assertions.assertNotNull(zipFile);
+ Assertions.assertNotNull(conf);
+ String zipFilePath = Paths.get(zipFile.toURI()).toString();
+ String confPath = Paths.get(conf.toURI()).toString();
+ Config pluginConfig =
+ ConfigFactory.parseString(
+ "poi_excel_max_file_size =
1\narchive_compress_codec = \"ZIP\"")
+ .withFallback(ConfigFactory.parseFile(new
File(confPath)));
+ ExcelReadStrategy excelReadStrategy = new ExcelReadStrategy();
+ LocalConf localConf = new LocalConf(FS_DEFAULT_NAME_DEFAULT);
+ excelReadStrategy.setPluginConfig(pluginConfig);
+ excelReadStrategy.init(localConf);
+
+ List<String> fileNamesByPath =
excelReadStrategy.getFileNamesByPath(zipFilePath);
+ CatalogTable userDefinedCatalogTable =
CatalogTableUtil.buildWithConfig(pluginConfig);
+ excelReadStrategy.setCatalogTable(userDefinedCatalogTable);
+
+ TestCollector testCollector = new TestCollector();
+ // EasyExcel streams rows lazily, so the POI size guard must NOT apply
+ // to archived entries even when the configured limit is below the
entry size.
+ excelReadStrategy.read(fileNamesByPath.get(0), "", testCollector);
+
+ Assertions.assertEquals(5, testCollector.getRows().size());
+ }
+
+ @Test
+ public void testPoiAcceptsSmallExcelEntryInsideLargerArchive()
+ throws IOException, URISyntaxException {
+ // Regression for the archived-path size guard: the on-disk size of the
+ // archive must NOT be used to judge an Excel entry bundled inside it.
Build
+ // a zip that is far larger than the configured POI limit on disk but
whose
+ // only Excel entry is well below the limit, then verify POI still
reads it.
+ // Previously the redundant POI-level guard re-statted the archive
path and
+ // falsely rejected such a small entry.
+ URL excelFile =
ExcelReadStrategyTest.class.getResource("/excel/e2e.xlsx");
+ URL conf =
ExcelReadStrategyTest.class.getResource("/excel/e2exls.conf");
+ Assertions.assertNotNull(excelFile);
+ Assertions.assertNotNull(conf);
+
+ byte[] excelBytes = Files.readAllBytes(Paths.get(excelFile.toURI()));
+ long entrySize = excelBytes.length;
+ // Limit above the Excel entry, but below the padded archive on disk.
+ long poiLimit = entrySize + 1024L;
+
+ File archive = File.createTempFile("e2e_padded_", ".zip");
+ archive.deleteOnExit();
+ // STORED (uncompressed) padding so the archive's on-disk size is
reliably
+ // larger than the limit regardless of how the Excel entry compresses.
+ byte[] padding = new byte[(int) (poiLimit * 4)];
+ CRC32 crc = new CRC32();
+ crc.update(padding);
+ try (ZipOutputStream zos = new ZipOutputStream(new
FileOutputStream(archive))) {
+ ZipEntry excelEntry = new ZipEntry("e2e.xlsx");
+ zos.putNextEntry(excelEntry);
+ zos.write(excelBytes);
+ zos.closeEntry();
+
+ ZipEntry paddingEntry = new ZipEntry("padding.bin");
+ paddingEntry.setMethod(ZipEntry.STORED);
+ paddingEntry.setSize(padding.length);
+ paddingEntry.setCompressedSize(padding.length);
+ paddingEntry.setCrc(crc.getValue());
+ zos.putNextEntry(paddingEntry);
+ zos.write(padding);
+ zos.closeEntry();
+ }
+
+ Assertions.assertTrue(archive.length() > poiLimit, "archive must
exceed the limit");
+ Assertions.assertTrue(entrySize < poiLimit, "entry must be below the
limit");
+
+ String confPath = Paths.get(conf.toURI()).toString();
+ Config pluginConfig =
+ ConfigFactory.parseString(
+ "poi_excel_max_file_size = "
+ + poiLimit
+ + "\narchive_compress_codec = \"ZIP\"")
+ .withFallback(ConfigFactory.parseFile(new
File(confPath)))
+ .withoutPath("excel_engine");
+ ExcelReadStrategy excelReadStrategy = new ExcelReadStrategy();
+ LocalConf localConf = new LocalConf(FS_DEFAULT_NAME_DEFAULT);
+ excelReadStrategy.setPluginConfig(pluginConfig);
+ excelReadStrategy.init(localConf);
+
+ List<String> fileNamesByPath =
+
excelReadStrategy.getFileNamesByPath(archive.getAbsolutePath());
+ CatalogTable userDefinedCatalogTable =
CatalogTableUtil.buildWithConfig(pluginConfig);
+ excelReadStrategy.setCatalogTable(userDefinedCatalogTable);
+
+ TestCollector testCollector = new TestCollector();
+ // POI engine (default). The archive exceeds the limit, but the Excel
entry
+ // is below it, so the read must succeed rather than be falsely
rejected.
+ excelReadStrategy.read(fileNamesByPath.get(0), "", testCollector);
+
+ Assertions.assertEquals(5, testCollector.getRows().size());
+ }
+
private void testLargeExcelRead(String filePath, String configPath, int
rowCount)
throws IOException, URISyntaxException {
URL excelFile = ExcelReadStrategyTest.class.getResource(filePath);
diff --git
a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/resources/excel/archive_zip/e2e_in_zip.zip
b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/resources/excel/archive_zip/e2e_in_zip.zip
new file mode 100644
index 0000000000..f70dd2607f
Binary files /dev/null and
b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/resources/excel/archive_zip/e2e_in_zip.zip
differ
diff --git
a/seatunnel-connectors-v2/connector-file/connector-file-cos/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/cos/source/CosFileSourceFactory.java
b/seatunnel-connectors-v2/connector-file/connector-file-cos/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/cos/source/CosFileSourceFactory.java
index e3efd70652..26ca989daa 100644
---
a/seatunnel-connectors-v2/connector-file/connector-file-cos/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/cos/source/CosFileSourceFactory.java
+++
b/seatunnel-connectors-v2/connector-file/connector-file-cos/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/cos/source/CosFileSourceFactory.java
@@ -67,10 +67,6 @@ public class CosFileSourceFactory implements
TableSourceFactory {
FileBaseSourceOptions.FILE_FORMAT_TYPE,
FileFormat.CSV,
FileBaseSourceOptions.SKIP_HEADER_ROW_NUMBER)
- .conditional(
- FileBaseSourceOptions.FILE_FORMAT_TYPE,
- FileFormat.EXCEL,
- FileBaseSourceOptions.SHEET_NAME)
.conditional(
FileBaseSourceOptions.FILE_FORMAT_TYPE,
Arrays.asList(
@@ -95,6 +91,10 @@ public class CosFileSourceFactory implements
TableSourceFactory {
.optional(FileBaseSourceOptions.NULL_FORMAT)
.optional(FileBaseSourceOptions.FILENAME_EXTENSION)
.optional(FileBaseSourceOptions.READ_COLUMNS)
+ .optional(
+ FileBaseSourceOptions.SHEET_NAME,
+ FileBaseSourceOptions.EXCEL_ENGINE,
+ FileBaseSourceOptions.POI_EXCEL_MAX_FILE_SIZE)
.conditional(
FileBaseSourceOptions.FILE_FORMAT_TYPE,
FileFormat.MARKDOWN,
diff --git
a/seatunnel-connectors-v2/connector-file/connector-file-ftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/ftp/source/FtpFileSourceFactory.java
b/seatunnel-connectors-v2/connector-file/connector-file-ftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/ftp/source/FtpFileSourceFactory.java
index 209e9b3eb4..269abc69a8 100644
---
a/seatunnel-connectors-v2/connector-file/connector-file-ftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/ftp/source/FtpFileSourceFactory.java
+++
b/seatunnel-connectors-v2/connector-file/connector-file-ftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/ftp/source/FtpFileSourceFactory.java
@@ -101,6 +101,10 @@ public class FtpFileSourceFactory implements
TableSourceFactory {
.optional(FileBaseSourceOptions.NULL_FORMAT)
.optional(FileBaseSourceOptions.FILENAME_EXTENSION)
.optional(FileBaseSourceOptions.READ_COLUMNS)
+ .optional(
+ FileBaseSourceOptions.SHEET_NAME,
+ FileBaseSourceOptions.EXCEL_ENGINE,
+ FileBaseSourceOptions.POI_EXCEL_MAX_FILE_SIZE)
.conditional(
FileBaseSourceOptions.FILE_FORMAT_TYPE,
FileFormat.MARKDOWN,
diff --git
a/seatunnel-connectors-v2/connector-file/connector-file-hadoop/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/hdfs/source/HdfsFileSourceFactory.java
b/seatunnel-connectors-v2/connector-file/connector-file-hadoop/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/hdfs/source/HdfsFileSourceFactory.java
index cad52a6931..e3bc352336 100644
---
a/seatunnel-connectors-v2/connector-file/connector-file-hadoop/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/hdfs/source/HdfsFileSourceFactory.java
+++
b/seatunnel-connectors-v2/connector-file/connector-file-hadoop/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/hdfs/source/HdfsFileSourceFactory.java
@@ -110,6 +110,10 @@ public class HdfsFileSourceFactory implements
TableSourceFactory {
.optional(HdfsFileSourceOptions.NULL_FORMAT)
.optional(HdfsFileSourceOptions.FILENAME_EXTENSION)
.optional(HdfsFileSourceOptions.READ_COLUMNS)
+ .optional(
+ FileBaseSourceOptions.SHEET_NAME,
+ FileBaseSourceOptions.EXCEL_ENGINE,
+ FileBaseSourceOptions.POI_EXCEL_MAX_FILE_SIZE)
.conditional(
HdfsFileSourceOptions.FILE_FORMAT_TYPE,
FileFormat.MARKDOWN,
diff --git
a/seatunnel-connectors-v2/connector-file/connector-file-jindo-oss/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/oss/jindo/source/OssFileSourceFactory.java
b/seatunnel-connectors-v2/connector-file/connector-file-jindo-oss/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/oss/jindo/source/OssFileSourceFactory.java
index 32bb57f6db..c833d2b041 100644
---
a/seatunnel-connectors-v2/connector-file/connector-file-jindo-oss/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/oss/jindo/source/OssFileSourceFactory.java
+++
b/seatunnel-connectors-v2/connector-file/connector-file-jindo-oss/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/oss/jindo/source/OssFileSourceFactory.java
@@ -90,6 +90,10 @@ public class OssFileSourceFactory implements
TableSourceFactory {
.optional(FileBaseSourceOptions.NULL_FORMAT)
.optional(FileBaseSourceOptions.FILENAME_EXTENSION)
.optional(FileBaseSourceOptions.READ_COLUMNS)
+ .optional(
+ FileBaseSourceOptions.SHEET_NAME,
+ FileBaseSourceOptions.EXCEL_ENGINE,
+ FileBaseSourceOptions.POI_EXCEL_MAX_FILE_SIZE)
.conditional(
FileBaseSourceOptions.FILE_FORMAT_TYPE,
FileFormat.MARKDOWN,
diff --git
a/seatunnel-connectors-v2/connector-file/connector-file-local/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/local/source/LocalFileSourceFactory.java
b/seatunnel-connectors-v2/connector-file/connector-file-local/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/local/source/LocalFileSourceFactory.java
index 6c50e98b87..de77146f4a 100644
---
a/seatunnel-connectors-v2/connector-file/connector-file-local/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/local/source/LocalFileSourceFactory.java
+++
b/seatunnel-connectors-v2/connector-file/connector-file-local/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/local/source/LocalFileSourceFactory.java
@@ -110,6 +110,10 @@ public class LocalFileSourceFactory implements
TableSourceFactory {
.optional(FileBaseSourceOptions.NULL_FORMAT)
.optional(FileBaseSourceOptions.FILENAME_EXTENSION)
.optional(FileBaseSourceOptions.READ_COLUMNS)
+ .optional(
+ FileBaseSourceOptions.SHEET_NAME,
+ FileBaseSourceOptions.EXCEL_ENGINE,
+ FileBaseSourceOptions.POI_EXCEL_MAX_FILE_SIZE)
.conditional(
FileBaseSourceOptions.FILE_FORMAT_TYPE,
FileFormat.MARKDOWN,
diff --git
a/seatunnel-connectors-v2/connector-file/connector-file-obs/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/obs/source/ObsFileSourceFactory.java
b/seatunnel-connectors-v2/connector-file/connector-file-obs/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/obs/source/ObsFileSourceFactory.java
index 0fdf4b4eaf..55e068f5d8 100644
---
a/seatunnel-connectors-v2/connector-file/connector-file-obs/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/obs/source/ObsFileSourceFactory.java
+++
b/seatunnel-connectors-v2/connector-file/connector-file-obs/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/obs/source/ObsFileSourceFactory.java
@@ -79,6 +79,10 @@ public class ObsFileSourceFactory implements
TableSourceFactory {
.optional(FileBaseSourceOptions.NULL_FORMAT)
.optional(FileBaseSourceOptions.FILENAME_EXTENSION)
.optional(FileBaseSourceOptions.READ_COLUMNS)
+ .optional(
+ FileBaseSourceOptions.SHEET_NAME,
+ FileBaseSourceOptions.EXCEL_ENGINE,
+ FileBaseSourceOptions.POI_EXCEL_MAX_FILE_SIZE)
.conditional(
FileBaseSourceOptions.FILE_FORMAT_TYPE,
FileFormat.MARKDOWN,
diff --git
a/seatunnel-connectors-v2/connector-file/connector-file-oss/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/oss/source/OssFileSourceFactory.java
b/seatunnel-connectors-v2/connector-file/connector-file-oss/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/oss/source/OssFileSourceFactory.java
index bfaa6a46da..943b02c3a1 100644
---
a/seatunnel-connectors-v2/connector-file/connector-file-oss/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/oss/source/OssFileSourceFactory.java
+++
b/seatunnel-connectors-v2/connector-file/connector-file-oss/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/oss/source/OssFileSourceFactory.java
@@ -99,6 +99,10 @@ public class OssFileSourceFactory implements
TableSourceFactory {
.optional(FileBaseSourceOptions.NULL_FORMAT)
.optional(FileBaseSourceOptions.FILENAME_EXTENSION)
.optional(FileBaseSourceOptions.READ_COLUMNS)
+ .optional(
+ FileBaseSourceOptions.SHEET_NAME,
+ FileBaseSourceOptions.EXCEL_ENGINE,
+ FileBaseSourceOptions.POI_EXCEL_MAX_FILE_SIZE)
.conditional(
FileBaseSourceOptions.FILE_FORMAT_TYPE,
FileFormat.MARKDOWN,
diff --git
a/seatunnel-connectors-v2/connector-file/connector-file-s3/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/s3/source/S3FileSourceFactory.java
b/seatunnel-connectors-v2/connector-file/connector-file-s3/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/s3/source/S3FileSourceFactory.java
index e0d5e1cf47..b30154c6ae 100644
---
a/seatunnel-connectors-v2/connector-file/connector-file-s3/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/s3/source/S3FileSourceFactory.java
+++
b/seatunnel-connectors-v2/connector-file/connector-file-s3/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/s3/source/S3FileSourceFactory.java
@@ -115,6 +115,10 @@ public class S3FileSourceFactory implements
TableSourceFactory {
.optional(FileBaseSourceOptions.NULL_FORMAT)
.optional(FileBaseSourceOptions.FILENAME_EXTENSION)
.optional(FileBaseSourceOptions.READ_COLUMNS)
+ .optional(
+ FileBaseSourceOptions.SHEET_NAME,
+ FileBaseSourceOptions.EXCEL_ENGINE,
+ FileBaseSourceOptions.POI_EXCEL_MAX_FILE_SIZE)
.conditional(
FileBaseSourceOptions.FILE_FORMAT_TYPE,
FileFormat.MARKDOWN,
diff --git
a/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/source/SftpFileSourceFactory.java
b/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/source/SftpFileSourceFactory.java
index f860d3bed3..160b260dc5 100644
---
a/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/source/SftpFileSourceFactory.java
+++
b/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/source/SftpFileSourceFactory.java
@@ -91,6 +91,10 @@ public class SftpFileSourceFactory implements
TableSourceFactory {
.optional(FileBaseSourceOptions.NULL_FORMAT)
.optional(FileBaseSourceOptions.FILENAME_EXTENSION)
.optional(FileBaseSourceOptions.READ_COLUMNS)
+ .optional(
+ FileBaseSourceOptions.SHEET_NAME,
+ FileBaseSourceOptions.EXCEL_ENGINE,
+ FileBaseSourceOptions.POI_EXCEL_MAX_FILE_SIZE)
.conditional(
FileBaseSourceOptions.FILE_FORMAT_TYPE,
FileFormat.MARKDOWN,
