This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/dev/pr-13141-7ed5e28f4a49b367f0fbed80e1bd4e351cfb0776 in repository https://gitbox.apache.org/repos/asf/seatunnel.git
commit d208f66bffa6452da7bc229af4970b9616850cd4 Author: Goutam Adwant <[email protected]> AuthorDate: Tue Nov 17 15:55:06 2026 +0000 [Feature][Connector-V2][Paimon] Support REST catalog for source and sink (#13141) Signed-off-by: Goutam Adwant <[email protected]> --- docs/en/connectors/sink/Paimon.md | 64 ++- docs/en/connectors/source/Paimon.md | 72 ++- docs/zh/connectors/sink/Paimon.md | 64 ++- docs/zh/connectors/source/Paimon.md | 72 ++- .../seatunnel/paimon/catalog/PaimonCatalog.java | 11 +- .../paimon/catalog/PaimonCatalogEnum.java | 3 +- .../paimon/catalog/PaimonCatalogFactory.java | 11 + .../paimon/catalog/PaimonCatalogLoader.java | 81 +++- .../seatunnel/paimon/config/PaimonBaseOptions.java | 43 +- .../seatunnel/paimon/config/PaimonConfig.java | 91 +++- .../seatunnel/paimon/sink/PaimonSinkFactory.java | 11 + .../paimon/source/PaimonSourceFactory.java | 11 + .../paimon/catalog/PaimonCatalogLoaderTest.java | 505 +++++++++++++++++++++ .../paimon/config/PaimonOptionRuleTest.java | 41 ++ .../core/starter/utils/ConfigShadeUtils.java | 2 + .../core/starter/utils/ConfigBuilderTest.java | 36 ++ .../connector-paimon-e2e/pom.xml | 25 + .../e2e/connector/paimon/PaimonRestCatalogIT.java | 331 ++++++++++++++ 18 files changed, 1447 insertions(+), 27 deletions(-) diff --git a/docs/en/connectors/sink/Paimon.md b/docs/en/connectors/sink/Paimon.md index e27ccd0300..0989e96f5b 100644 --- a/docs/en/connectors/sink/Paimon.md +++ b/docs/en/connectors/sink/Paimon.md @@ -70,10 +70,16 @@ libfb303-xxx.jar | name | type | required | default value | Description | |------------------------------|---------|----------|------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| warehouse | String | Yes | - | Paimon warehouse path. Must not be blank. | +| warehouse | String | Yes | - | Paimon warehouse path. Must not be blank. For `rest`, use the managed warehouse name instead of a path. | | catalog_name | String | No | paimon | The name of the Paimon catalog. | -| catalog_type | String | No | filesystem | Catalog type of Paimon. Supports `filesystem` and `hive`. | -| catalog_uri | String | Yes when `catalog_type` is `hive` | - | Catalog URI of Paimon. Required when `catalog_type` is `hive`. | +| catalog_type | String | No | filesystem | Catalog type of Paimon. Supports `filesystem`, `hive` and `rest`. | +| catalog_uri | String | Yes when `catalog_type` is `hive` or `rest` | - | Catalog URI of Paimon: the Hive metastore URI, or the REST catalog endpoint. Required when `catalog_type` is `hive` or `rest`. | +| token.provider | String | Yes when `catalog_type` is `rest` | - | Authentication provider for the REST catalog: exactly `bear` or `dlf`. | +| token | String | Yes when `token.provider` is `bear` | - | Bearer token for the REST catalog. | +| dlf.access-key-id | String | Yes when `token.provider` is `dlf` | - | DLF access key ID. | +| dlf.access-key-secret | String | Yes when `token.provider` is `dlf` | - | DLF access key secret. | +| dlf.security-token | String | No | - | Optional temporary DLF security token. | +| dlf.region | String | No | - | DLF region. Inferred from `catalog_uri` when omitted. | | database | String | Yes | - | The database you want to access. Must not be blank. | | table | String | Yes | - | The table you want to access. Must not be blank. | | user | String | No | - | The Paimon user used to access the table. Required when the underlying catalog enforces authentication. | @@ -177,6 +183,58 @@ CDC ingestion supports a limited number of schema changes. The currently support ## Examples + +### REST catalog authentication + +`catalog_type = "rest"` uses the REST implementation included in Paimon 1.1.1. Set `catalog_uri` to the HTTPS catalog endpoint and `warehouse` to the managed warehouse **name**, not an HDFS/S3 path. The REST server supplies table locations; workers still need access and credentials for the underlying storage. Existing filesystem and Hive configurations are unchanged. + +For REST, `catalog_uri` must be an HTTP or HTTPS URL with a host and no user information. Use HTTPS outside loopback testing: HTTP to a non-loopback host produces a warning because authentication credentials travel in clear text. `token.provider` is required and accepts exactly `bear` or `dlf`. `bear` requires a non-blank `token`; `dlf` requires both `dlf.access-key-id` and `dlf.access-key-secret`. + +Bearer tokens and DLF credentials are masked in parsed-configuration logs. For configuration encryption, `token` is included in the default `shade.identifier` handling; add the DLF secret keys explicitly to `shade.options` to encrypt and decrypt them. Log masking alone does not encrypt configuration. Do not put credentials in endpoint URLs. Supply secrets through environment variables or your deployment's secret mechanism. + +Only explicitly configured Hadoop storage properties are forwarded to REST. Catalog and authentication keys (`uri`, `warehouse`, `metastore`, `token`, `token.*`, `dlf.*`, including `hadoop.`-prefixed forms) are rejected in `paimon.hadoop.conf` and the configured Hadoop XML files. Configure REST authentication with the top-level options shown below. `dlf.token-path`, `dlf.token-loader`, `dlf.token-ecs-*`, and `token.refresh-time` are not supported in this version and cannot be enabled thr [...] + +Parsed-configuration masking does not redact dependency logs. Paimon 1.1.1 logs Hadoop storage-property values, which may include credentials, when `org.apache.paimon.utils.HadoopUtils` DEBUG logging is enabled. Keep this logger at INFO or higher when using credential-bearing storage configuration. + +Paimon 1.1.1's shared OkHttp client follows cross-host redirects and forwards `x-dlf-security-token`, although it strips bearer `Authorization`. Use trusted catalog endpoints that do not redirect to another host; do not rely on redirects to protect DLF temporary credentials. This dependency limitation is not changed by the connector. + +Bearer example: + +```hocon +sink { + Paimon { + catalog_type = "rest" + catalog_uri = "https://catalog.example.com" + warehouse = "managed-warehouse" + database = "example_db" + table = "example_table" + "token.provider" = "bear" + token = ${PAIMON_REST_TOKEN} + } +} +``` + +DLF example (replace `catalog_uri` with your DLF endpoint): + +```hocon +sink { + Paimon { + catalog_type = "rest" + catalog_uri = "https://catalog.example.com" + warehouse = "managed-warehouse" + database = "example_db" + table = "example_table" + "token.provider" = "dlf" + "dlf.access-key-id" = ${DLF_ACCESS_KEY_ID} + "dlf.access-key-secret" = ${DLF_ACCESS_KEY_SECRET} + "dlf.security-token" = ${?DLF_SECURITY_TOKEN} + "dlf.region" = "cn-hangzhou" + } +} +``` + +`dlf.security-token` is optional and only needed for temporary credentials. `dlf.region` may be omitted if Paimon can infer it from the DLF endpoint. The connector forwards static credentials; it does not implement automatic refresh of temporary credentials. Live DLF authentication and cloud-storage permissions must be verified in the target deployment. + ### Schema evolution ```hocon env { diff --git a/docs/en/connectors/source/Paimon.md b/docs/en/connectors/source/Paimon.md index 3b37e7a547..51776f6ee1 100644 --- a/docs/en/connectors/source/Paimon.md +++ b/docs/en/connectors/source/Paimon.md @@ -54,10 +54,16 @@ Read data from Apache Paimon. | name | type | required | default value | description | |-------------------------|----------|----------------|---------------|------------------------------------------------------------------------------------------------------------------------------| -| warehouse | String | Yes | - | Paimon warehouse path. | +| warehouse | String | Yes | - | Paimon warehouse path. For `rest`, use the managed warehouse name instead of a path. | | catalog_name | String | No | paimon | The name of the Paimon catalog. | -| catalog_type | String | No | filesystem | Catalog type of Paimon, supports `filesystem` and `hive`. | -| catalog_uri | String | Yes when `catalog_type` is `hive` | - | Catalog URI of Paimon. Required when `catalog_type` is `hive`. | +| catalog_type | String | No | filesystem | Catalog type of Paimon, supports `filesystem`, `hive` and `rest`. | +| catalog_uri | String | Yes when `catalog_type` is `hive` or `rest` | - | Catalog URI of Paimon: the Hive metastore URI, or the REST catalog endpoint. Required when `catalog_type` is `hive` or `rest`. | +| token.provider | String | Yes when `catalog_type` is `rest` | - | Authentication provider for the REST catalog: exactly `bear` or `dlf`. | +| token | String | Yes when `token.provider` is `bear` | - | Bearer token for the REST catalog. | +| dlf.access-key-id | String | Yes when `token.provider` is `dlf` | - | DLF access key ID. | +| dlf.access-key-secret | String | Yes when `token.provider` is `dlf` | - | DLF access key secret. | +| dlf.security-token | String | No | - | Optional temporary DLF security token. | +| dlf.region | String | No | - | DLF region. Inferred from `catalog_uri` when omitted. | | database | String | Yes | - | The database you want to access. | | table | String | Yes when `table_list` is absent | - | The table you want to access. Configure exactly one of `table` and `table_list`. | | table_list | array | Yes when `table` is absent | - | The list of tables to read. Each item must contain `database` and `table`, and may contain its own `query`. | @@ -70,7 +76,7 @@ Read data from Apache Paimon. ### warehouse [string] -Paimon warehouse path. Must not be blank. +Paimon warehouse path, or the managed warehouse name for REST catalogs. Must not be blank. ### catalog_name [string] @@ -78,11 +84,11 @@ The name of the Paimon catalog. Default value is `paimon`. ### catalog_type [string] -Catalog type of Paimon. Supports `filesystem` and `hive`. Default value is `filesystem`. +Catalog type of Paimon. Supports `filesystem`, `hive` and `rest`. Default value is `filesystem`. ### catalog_uri [string] -Catalog URI of Paimon. This option is required when `catalog_type` is `hive` (for example, `thrift://hadoop04:9083`). +Catalog URI of Paimon. Required when `catalog_type` is `hive` (for example, `thrift://hadoop04:9083`) or `rest` (the REST catalog endpoint, for example `https://dlf.cn-hangzhou.aliyuncs.com`). ### database [string] @@ -164,6 +170,58 @@ Besides, the warehouse should start with `s3a://`. ## Examples +### REST catalog authentication + +`catalog_type = "rest"` uses the REST implementation included in Paimon 1.1.1. Set `catalog_uri` to the HTTPS catalog endpoint and `warehouse` to the managed warehouse **name**, not an HDFS/S3 path. The REST server supplies table locations; workers still need access and credentials for the underlying storage. Existing filesystem and Hive configurations are unchanged. + +For REST, `catalog_uri` must be an HTTP or HTTPS URL with a host and no user information. Use HTTPS outside loopback testing: HTTP to a non-loopback host produces a warning because authentication credentials travel in clear text. `token.provider` is required and accepts exactly `bear` or `dlf`. `bear` requires a non-blank `token`; `dlf` requires both `dlf.access-key-id` and `dlf.access-key-secret`. + +Bearer tokens and DLF credentials are masked in parsed-configuration logs. For configuration encryption, `token` is included in the default `shade.identifier` handling; add the DLF secret keys explicitly to `shade.options` to encrypt and decrypt them. Log masking alone does not encrypt configuration. Do not put credentials in endpoint URLs. Supply secrets through environment variables or your deployment's secret mechanism. + +Only explicitly configured Hadoop storage properties are forwarded to REST. Catalog and authentication keys (`uri`, `warehouse`, `metastore`, `token`, `token.*`, `dlf.*`, including `hadoop.`-prefixed forms) are rejected in `paimon.hadoop.conf` and the configured Hadoop XML files. Configure REST authentication with the top-level options shown below. `dlf.token-path`, `dlf.token-loader`, `dlf.token-ecs-*`, and `token.refresh-time` are not supported in this version and cannot be enabled thr [...] + +Parsed-configuration masking does not redact dependency logs. Paimon 1.1.1 logs Hadoop storage-property values, which may include credentials, when `org.apache.paimon.utils.HadoopUtils` DEBUG logging is enabled. Keep this logger at INFO or higher when using credential-bearing storage configuration. + +Paimon 1.1.1's shared OkHttp client follows cross-host redirects and forwards `x-dlf-security-token`, although it strips bearer `Authorization`. Use trusted catalog endpoints that do not redirect to another host; do not rely on redirects to protect DLF temporary credentials. This dependency limitation is not changed by the connector. + +Bearer example: + +```hocon +source { + Paimon { + catalog_type = "rest" + catalog_uri = "https://catalog.example.com" + warehouse = "managed-warehouse" + database = "example_db" + table = "example_table" + "token.provider" = "bear" + token = ${PAIMON_REST_TOKEN} + } +} +``` + +DLF example (replace `catalog_uri` with your DLF endpoint): + +```hocon +source { + Paimon { + catalog_type = "rest" + catalog_uri = "https://catalog.example.com" + warehouse = "managed-warehouse" + database = "example_db" + table = "example_table" + "token.provider" = "dlf" + "dlf.access-key-id" = ${DLF_ACCESS_KEY_ID} + "dlf.access-key-secret" = ${DLF_ACCESS_KEY_SECRET} + "dlf.security-token" = ${?DLF_SECURITY_TOKEN} + "dlf.region" = "cn-hangzhou" + } +} +``` + +`dlf.security-token` is optional and only needed for temporary credentials. `dlf.region` may be omitted if Paimon can infer it from the DLF endpoint. The connector forwards static credentials; it does not implement automatic refresh of temporary credentials. Live DLF authentication and cloud-storage permissions must be verified in the target deployment. + + ### Simple example ```hocon @@ -344,6 +402,8 @@ The SeaTunnel Paimon source supports both batch snapshot queries and streaming c Configure `warehouse` pointing to the storage root (e.g. `hdfs:///paimon/warehouse`, `s3a://bucket/warehouse`, or local path) and set `catalog_type` (such as `filesystem` or `hive`). Required storage or authentication properties can be provided via `paimon.hadoop.conf`. +For `catalog_type = "rest"`, use the managed warehouse name instead of a storage path and set `catalog_uri` and the catalog authentication options shown above. Storage properties still belong in `paimon.hadoop.conf`; they are separate from REST catalog credentials. + ### Does Paimon source support column projection? Yes. The Paimon source reads only the projection fields defined in the SeaTunnel job schema, avoiding unnecessary column deserialization and optimizing read performance on columnar formats like ORC and Parquet. diff --git a/docs/zh/connectors/sink/Paimon.md b/docs/zh/connectors/sink/Paimon.md index 57535c0a8f..4ca9f161b5 100644 --- a/docs/zh/connectors/sink/Paimon.md +++ b/docs/zh/connectors/sink/Paimon.md @@ -70,10 +70,16 @@ libfb303-xxx.jar | 名称 | 类型 | 是否必须 | 默认值 | 描述 | |------------------------------|------|------|------------------------------|------------------------------------------------------------------------------------------------------| -| warehouse | 字符串 | 是 | - | Paimon warehouse路径,不能为空白字符串。 | +| warehouse | 字符串 | 是 | - | warehouse 路径;REST catalog 使用托管 warehouse 名称,而不是存储路径。 | | catalog_name | 字符串 | 否 | paimon | Paimon catalog名称 | -| catalog_type | 字符串 | 否 | filesystem | Paimon的catalog类型,目前支持filesystem和hive | -| catalog_uri | 字符串 | 当 `catalog_type` 为 `hive` 时是 | - | Paimon catalog 的 URI。当 `catalog_type` 为 `hive` 时必须配置。 | +| catalog_type | 字符串 | 否 | filesystem | catalog 类型:`filesystem`、`hive` 或 `rest`。 | +| catalog_uri | 字符串 | `hive` 或 `rest` 时必填 | - | Hive metastore URI 或 REST catalog 服务端点。 | +| token.provider | String | `rest` 时必填 | - | 认证提供者:仅小写 `bear` 或 `dlf`。 | +| token | String | `bear` 时必填 | - | Bearer token。 | +| dlf.access-key-id | String | `dlf` 时必填 | - | DLF access key ID。 | +| dlf.access-key-secret | String | `dlf` 时必填 | - | DLF access key secret。 | +| dlf.security-token | String | 否 | - | 可选的临时 DLF security token。 | +| dlf.region | String | 否 | - | DLF region;未配置时从 `catalog_uri` 推断。 | | database | 字符串 | 是 | - | 数据库名称,不能为空白字符串。 | | table | 字符串 | 是 | - | 表名,不能为空白字符串。 | | user | 字符串 | 否 | - | paimon开启权限后,用户名 | @@ -164,6 +170,58 @@ Cdc采集支持有限数量的模式更改。目前支持的模式更改包括 ## 示例 +### REST catalog 认证 + +`catalog_type = "rest"` 使用 Paimon 1.1.1 内置的 REST 实现。`catalog_uri` 为 HTTPS catalog 端点,`warehouse` 为托管 warehouse **名称**,而不是 HDFS/S3 路径。表的存储位置由 REST 服务返回;worker 仍需具有底层存储的访问权限和凭证。现有 filesystem 和 Hive 配置保持不变。 + +REST 的 `catalog_uri` 必须是包含主机且不含用户信息的 HTTP 或 HTTPS URL。除回环地址测试外,请使用 HTTPS:非回环主机使用 HTTP 会产生警告,因为认证凭证会明文传输。`token.provider` 必填,且仅接受小写 `bear` 或 `dlf`。`bear` 要求非空白的 `token`;`dlf` 要求同时配置 `dlf.access-key-id` 和 `dlf.access-key-secret`。 + +Bearer token 和 DLF 凭证在解析后的配置日志中自动脱敏。配置加密时,`token` 默认包含在 `shade.identifier` 的处理范围内;DLF 密钥必须显式加入 `shade.options` 才能加密和解密。日志脱敏本身不会加密配置。不要在端点 URL 中放置凭证。请通过环境变量或部署环境的密钥机制传入。 + +REST 仅转发显式配置的 Hadoop 存储参数。`paimon.hadoop.conf` 和指定的 Hadoop XML 文件中禁止配置 catalog 和认证键(`uri`、`warehouse`、`metastore`、`token`、`token.*`、`dlf.*`,包括带 `hadoop.` 前缀的形式)。请通过下方的顶层选项配置 REST 认证。此版本不支持 `dlf.token-path`、`dlf.token-loader`、`dlf.token-ecs-*` 和 `token.refresh-time`,也不能通过 `paimon.hadoop.conf` 启用它们。 + +解析后的配置脱敏不会过滤依赖库日志。Paimon 1.1.1 在 `org.apache.paimon.utils.HadoopUtils` 启用 DEBUG 日志时会记录 Hadoop 存储参数的值,其中可能包含凭据。使用包含凭据的存储配置时,请将此日志级别保持为 INFO 或更高。 + +Paimon 1.1.1 的共享 OkHttp 客户端会跟随跨主机重定向并转发 `x-dlf-security-token`,但会移除 bearer `Authorization`。请使用可信且不会重定向到其他主机的 catalog 端点,不要依赖重定向保护 DLF 临时凭证。此连接器没有修改该依赖库限制。 + +Bearer 示例: + +```hocon +sink { + Paimon { + catalog_type = "rest" + catalog_uri = "https://catalog.example.com" + warehouse = "managed-warehouse" + database = "example_db" + table = "example_table" + "token.provider" = "bear" + token = ${PAIMON_REST_TOKEN} + } +} +``` + +DLF 示例(将 `catalog_uri` 替换为实际 DLF 端点): + +```hocon +sink { + Paimon { + catalog_type = "rest" + catalog_uri = "https://catalog.example.com" + warehouse = "managed-warehouse" + database = "example_db" + table = "example_table" + "token.provider" = "dlf" + "dlf.access-key-id" = ${DLF_ACCESS_KEY_ID} + "dlf.access-key-secret" = ${DLF_ACCESS_KEY_SECRET} + "dlf.security-token" = ${?DLF_SECURITY_TOKEN} + "dlf.region" = "cn-hangzhou" + } +} +``` + +`dlf.security-token` 是可选参数,仅临时凭证需要。若 Paimon 可从 DLF 端点推断 region,则可省略 `dlf.region`。此实现传递静态凭证,不提供临时凭证自动刷新。请在目标部署环境中验证真实 DLF 认证及云存储权限。 + + ### 模式演变 ```hocon env { diff --git a/docs/zh/connectors/source/Paimon.md b/docs/zh/connectors/source/Paimon.md index 662d8da2b5..274c39f607 100644 --- a/docs/zh/connectors/source/Paimon.md +++ b/docs/zh/connectors/source/Paimon.md @@ -54,10 +54,16 @@ import ChangeLog from '../changelog/connector-paimon.md'; | 名称 | 类型 | 是否必须 | 默认值 | 描述 | |-------------------------|----------|--------|---------------|---| -| warehouse | String | 是 | - | Paimon warehouse 路径,不能为空白字符串。 | +| warehouse | String | 是 | - | Paimon warehouse 路径,不能为空白字符串;`rest` 类型使用托管 warehouse 名称而不是路径。 | | catalog_name | String | 否 | paimon | Paimon Catalog 名称。 | -| catalog_type | String | 否 | filesystem | Paimon Catalog 类型,支持 `filesystem` 和 `hive`。 | -| catalog_uri | String | 当 `catalog_type` 为 `hive` 时是 | - | Paimon Catalog URI;`catalog_type` 为 `hive` 时必须配置。 | +| catalog_type | String | 否 | filesystem | Paimon Catalog 类型,支持 `filesystem`、`hive` 和 `rest`。 | +| catalog_uri | String | 当 `catalog_type` 为 `hive` 或 `rest` 时是 | - | Paimon Catalog URI:Hive Metastore 地址或 REST Catalog 端点;`catalog_type` 为 `hive` 或 `rest` 时必须配置。 | +| token.provider | String | `catalog_type` 为 `rest` 时是 | - | REST Catalog 的认证方式,只能是 `bear` 或 `dlf`。 | +| token | String | `token.provider` 为 `bear` 时是 | - | REST Catalog 的 Bearer token。 | +| dlf.access-key-id | String | `token.provider` 为 `dlf` 时是 | - | DLF 的 Access Key ID。 | +| dlf.access-key-secret | String | `token.provider` 为 `dlf` 时是 | - | DLF 的 Access Key Secret。 | +| dlf.security-token | String | 否 | - | 可选的 DLF 临时安全令牌。 | +| dlf.region | String | 否 | - | DLF 区域;未配置时从 `catalog_uri` 推断。 | | database | String | 是 | - | 需要访问的数据库。 | | table | String | 未配置 `table_list` 时是 | - | 需要访问的表;必须在 `table` 和 `table_list` 中选择一个配置。 | | table_list | array | 未配置 `table` 时是 | - | 需要读取的表列表;每项必须包含 `database` 和 `table`,可单独配置 `query`。 | @@ -70,7 +76,7 @@ import ChangeLog from '../changelog/connector-paimon.md'; ### warehouse [string] -Paimon warehouse 路径,不能为空白字符串。 +Paimon warehouse 路径;REST catalog 使用托管 warehouse 名称。不能为空白字符串。 ### catalog_name [string] @@ -78,11 +84,11 @@ Paimon catalog 的名称。默认值为 `paimon`。 ### catalog_type [string] -Paimon Catalog 类型,支持 filesystem 和 hive +Paimon Catalog 类型,支持 `filesystem`、`hive` 和 `rest`。 ### catalog_uri [string] -Paimon 的 catalog URI。当 `catalog_type` 为 `hive` 时必须配置。 +Hive metastore URI 或 REST catalog 端点。当 `catalog_type` 为 `hive` 或 `rest` 时必须配置。 ### database [string] @@ -155,6 +161,58 @@ Paimon 连接器支持从多个文件系统读取数据。目前,支持的文 ## 示例 +### REST catalog 认证 + +`catalog_type = "rest"` 使用 Paimon 1.1.1 内置的 REST 实现。`catalog_uri` 为 HTTPS catalog 端点,`warehouse` 为托管 warehouse **名称**,而不是 HDFS/S3 路径。表的存储位置由 REST 服务返回;worker 仍需具有底层存储的访问权限和凭证。现有 filesystem 和 Hive 配置保持不变。 + +REST 的 `catalog_uri` 必须是包含主机且不含用户信息的 HTTP 或 HTTPS URL。除回环地址测试外,请使用 HTTPS:非回环主机使用 HTTP 会产生警告,因为认证凭证会明文传输。`token.provider` 必填,且仅接受小写 `bear` 或 `dlf`。`bear` 要求非空白的 `token`;`dlf` 要求同时配置 `dlf.access-key-id` 和 `dlf.access-key-secret`。 + +Bearer token 和 DLF 凭证在解析后的配置日志中自动脱敏。配置加密时,`token` 默认包含在 `shade.identifier` 的处理范围内;DLF 密钥必须显式加入 `shade.options` 才能加密和解密。日志脱敏本身不会加密配置。不要在端点 URL 中放置凭证。请通过环境变量或部署环境的密钥机制传入。 + +REST 仅转发显式配置的 Hadoop 存储参数。`paimon.hadoop.conf` 和指定的 Hadoop XML 文件中禁止配置 catalog 和认证键(`uri`、`warehouse`、`metastore`、`token`、`token.*`、`dlf.*`,包括带 `hadoop.` 前缀的形式)。请通过下方的顶层选项配置 REST 认证。此版本不支持 `dlf.token-path`、`dlf.token-loader`、`dlf.token-ecs-*` 和 `token.refresh-time`,也不能通过 `paimon.hadoop.conf` 启用它们。 + +解析后的配置脱敏不会过滤依赖库日志。Paimon 1.1.1 在 `org.apache.paimon.utils.HadoopUtils` 启用 DEBUG 日志时会记录 Hadoop 存储参数的值,其中可能包含凭据。使用包含凭据的存储配置时,请将此日志级别保持为 INFO 或更高。 + +Paimon 1.1.1 的共享 OkHttp 客户端会跟随跨主机重定向并转发 `x-dlf-security-token`,但会移除 bearer `Authorization`。请使用可信且不会重定向到其他主机的 catalog 端点,不要依赖重定向保护 DLF 临时凭证。此连接器没有修改该依赖库限制。 + +Bearer 示例: + +```hocon +source { + Paimon { + catalog_type = "rest" + catalog_uri = "https://catalog.example.com" + warehouse = "managed-warehouse" + database = "example_db" + table = "example_table" + "token.provider" = "bear" + token = ${PAIMON_REST_TOKEN} + } +} +``` + +DLF 示例(将 `catalog_uri` 替换为实际 DLF 端点): + +```hocon +source { + Paimon { + catalog_type = "rest" + catalog_uri = "https://catalog.example.com" + warehouse = "managed-warehouse" + database = "example_db" + table = "example_table" + "token.provider" = "dlf" + "dlf.access-key-id" = ${DLF_ACCESS_KEY_ID} + "dlf.access-key-secret" = ${DLF_ACCESS_KEY_SECRET} + "dlf.security-token" = ${?DLF_SECURITY_TOKEN} + "dlf.region" = "cn-hangzhou" + } +} +``` + +`dlf.security-token` 是可选参数,仅临时凭证需要。若 Paimon 可从 DLF 端点推断 region,则可省略 `dlf.region`。此实现传递静态凭证,不提供临时凭证自动刷新。请在目标部署环境中验证真实 DLF 认证及云存储权限。 + + ### 简单示例 ```hocon @@ -335,6 +393,8 @@ SeaTunnel Paimon 源连接器支持批处理快照读取与流式 changelog 消 通过 `warehouse` 配置底层存储根路径(如 `hdfs:///paimon/warehouse`、`s3a://bucket/warehouse` 或本地路径),并通过 `catalog_type` 指定 Catalog 类型(如 `filesystem`、`hive` 等)。相关的存储认证参数可统一在 `paimon.hadoop.conf` 中声明。 +使用 `catalog_type = "rest"` 时,`warehouse` 应为托管仓库名称而非存储路径,并按上述示例配置 `catalog_uri` 和 Catalog 认证参数。底层存储参数仍通过 `paimon.hadoop.conf` 配置,与 REST Catalog 凭据分开。 + ### Paimon 源连接器是否支持列投影? 支持。Paimon 源连接器仅读取上游作业 Schema 中声明的目标字段,对底层 ORC 或 Parquet 等列式存储格式避免冗余数据反序列化,从而提升查询性能。 diff --git a/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/catalog/PaimonCatalog.java b/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/catalog/PaimonCatalog.java index d169380047..91eba83889 100644 --- a/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/catalog/PaimonCatalog.java +++ b/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/catalog/PaimonCatalog.java @@ -32,6 +32,7 @@ import org.apache.seatunnel.api.table.catalog.exception.TableAlreadyExistExcepti import org.apache.seatunnel.api.table.catalog.exception.TableNotExistException; import org.apache.seatunnel.api.table.converter.BasicTypeDefine; import org.apache.seatunnel.common.constants.PluginType; +import org.apache.seatunnel.connectors.seatunnel.paimon.config.PaimonBaseOptions; import org.apache.seatunnel.connectors.seatunnel.paimon.config.PaimonConfig; import org.apache.seatunnel.connectors.seatunnel.paimon.config.PaimonSinkConfig; import org.apache.seatunnel.connectors.seatunnel.paimon.exception.PaimonConnectorErrorCode; @@ -52,7 +53,6 @@ import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import java.io.Closeable; -import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -85,10 +85,13 @@ public class PaimonCatalog implements Catalog, PaimonTable { @Override public void close() throws CatalogException { - if (catalog != null && catalog instanceof Closeable) { + if (catalog != null + && (catalog instanceof Closeable + || readonlyConfig.get(PaimonBaseOptions.CATALOG_TYPE) + == PaimonCatalogEnum.REST)) { try { - ((Closeable) catalog).close(); - } catch (IOException e) { + catalog.close(); + } catch (Exception e) { log.error("Error while closing PaimonCatalog.", e); throw new CatalogException(e); } diff --git a/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/catalog/PaimonCatalogEnum.java b/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/catalog/PaimonCatalogEnum.java index a90bd729f4..b3489a0d70 100644 --- a/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/catalog/PaimonCatalogEnum.java +++ b/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/catalog/PaimonCatalogEnum.java @@ -19,7 +19,8 @@ package org.apache.seatunnel.connectors.seatunnel.paimon.catalog; public enum PaimonCatalogEnum { FILESYSTEM("filesystem"), - HIVE("hive"); + HIVE("hive"), + REST("rest"); final String type; diff --git a/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/catalog/PaimonCatalogFactory.java b/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/catalog/PaimonCatalogFactory.java index 35629fa7d6..d8e5146a42 100644 --- a/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/catalog/PaimonCatalogFactory.java +++ b/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/catalog/PaimonCatalogFactory.java @@ -57,6 +57,12 @@ public class PaimonCatalogFactory implements CatalogFactory { PaimonBaseOptions.HADOOP_CONF, PaimonBaseOptions.HADOOP_CONF_PATH, PaimonBaseOptions.CATALOG_TYPE, + PaimonBaseOptions.TOKEN_PROVIDER, + PaimonBaseOptions.TOKEN, + PaimonBaseOptions.DLF_ACCESS_KEY_ID, + PaimonBaseOptions.DLF_ACCESS_KEY_SECRET, + PaimonBaseOptions.DLF_SECURITY_TOKEN, + PaimonBaseOptions.DLF_REGION, PaimonBaseOptions.USER, PaimonBaseOptions.PASSWORD, PaimonSinkOptions.SCHEMA_SAVE_MODE, @@ -75,6 +81,11 @@ public class PaimonCatalogFactory implements CatalogFactory { PaimonBaseOptions.CATALOG_TYPE, PaimonCatalogEnum.HIVE, PaimonBaseOptions.CATALOG_URI) + .conditional( + PaimonBaseOptions.CATALOG_TYPE, + PaimonCatalogEnum.REST, + PaimonBaseOptions.CATALOG_URI, + PaimonBaseOptions.TOKEN_PROVIDER) .build(); } } diff --git a/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/catalog/PaimonCatalogLoader.java b/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/catalog/PaimonCatalogLoader.java index 37e71108d7..f4947391d1 100644 --- a/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/catalog/PaimonCatalogLoader.java +++ b/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/catalog/PaimonCatalogLoader.java @@ -38,6 +38,8 @@ import org.apache.paimon.privilege.PrivilegedCatalog; import lombok.extern.slf4j.Slf4j; import java.io.Serializable; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.HashMap; import java.util.Iterator; import java.util.Map; @@ -63,19 +65,29 @@ public class PaimonCatalogLoader implements Serializable { private PaimonHadoopConfiguration paimonHadoopConfiguration; protected String user; protected String password; + private final Map<String, String> restOptions; + private final Map<String, String> restStorageOptions; public PaimonCatalogLoader(PaimonConfig paimonConfig) { this.warehouse = paimonConfig.getWarehouse(); this.catalogType = paimonConfig.getCatalogType(); this.catalogUri = paimonConfig.getCatalogUri(); + this.restStorageOptions = + catalogType == PaimonCatalogEnum.REST + ? loadRestStorageOptions(paimonConfig) + : new HashMap<>(); this.paimonHadoopConfiguration = PaimonSecurityContext.loadHadoopConfig(paimonConfig); this.user = paimonConfig.getUser(); this.password = paimonConfig.getPassword(); + this.restOptions = new HashMap<>(paimonConfig.getRestOptions()); } public Catalog loadCatalog() { // When using the seatunnel engine, set the current class loader to prevent loading failures Thread.currentThread().setContextClassLoader(PaimonCatalogLoader.class.getClassLoader()); + if (catalogType == PaimonCatalogEnum.REST) { + return loadRestCatalog(); + } final Map<String, String> optionsMap = new HashMap<>(1); optionsMap.put(CatalogOptions.WAREHOUSE.key(), warehouse); optionsMap.put(CatalogOptions.METASTORE.key(), catalogType.getType()); @@ -97,8 +109,75 @@ public class PaimonCatalogLoader implements Serializable { optionsMap.put(CatalogOptions.URI.key(), catalogUri); optionsMap.putAll(paimonHadoopConfiguration.getPropsWithPrefix(StringUtils.EMPTY)); } - final Options options = Options.fromMap(optionsMap); PaimonSecurityContext.shouldEnableKerberos(paimonHadoopConfiguration); + return createCatalog(optionsMap); + } + + private Catalog loadRestCatalog() { + Map<String, String> storageOptions = new HashMap<>(restStorageOptions); + PaimonSecurityContext.shouldEnableKerberos(paimonHadoopConfiguration); + if ("kerberos".equals(paimonHadoopConfiguration.get("hadoop.security.authentication"))) { + storageOptions.put("hadoop.security.authentication", "kerberos"); + } + Map<String, String> optionsMap = new HashMap<>(storageOptions); + // RESTCatalog rebuilds its context from options after fetching server configuration. + storageOptions.forEach((key, value) -> optionsMap.put("hadoop." + key, value)); + optionsMap.put(CatalogOptions.WAREHOUSE.key(), warehouse); + optionsMap.put(CatalogOptions.METASTORE.key(), catalogType.getType()); + optionsMap.put(CatalogOptions.URI.key(), catalogUri); + optionsMap.putAll(restOptions); + if (StringUtils.isNotBlank(user) && StringUtils.isNotBlank(password)) { + optionsMap.put(PaimonBaseOptions.USER.key(), user); + optionsMap.put(PaimonBaseOptions.PASSWORD.key(), password); + } + return createCatalog(optionsMap); + } + + private Map<String, String> loadRestStorageOptions(PaimonConfig config) { + Configuration explicit = new Configuration(false); + if (StringUtils.isNotBlank(config.getHdfsSitePath())) { + explicit.addResource(new org.apache.hadoop.fs.Path(config.getHdfsSitePath())); + } + if (StringUtils.isNotBlank(config.getHadoopConfPath())) { + for (String name : new String[] {"core-site.xml", "hdfs-site.xml", "hive-site.xml"}) { + java.nio.file.Path path = Paths.get(config.getHadoopConfPath(), name); + if (Files.exists(path)) { + explicit.addResource(new org.apache.hadoop.fs.Path(path.toUri())); + } + } + } + Map<String, String> storage = new HashMap<>(); + for (Map.Entry<String, String> entry : explicit) { + validateRestStorageKey(entry.getKey()); + storage.put(entry.getKey(), entry.getValue()); + } + config.getHadoopConfProps() + .forEach( + (key, value) -> { + validateRestStorageKey(key); + storage.put(key, value); + }); + return storage; + } + + private void validateRestStorageKey(String key) { + String option = key; + while (option.startsWith("hadoop.")) { + option = option.substring("hadoop.".length()); + } + if (option.equals("uri") + || option.equals("warehouse") + || option.equals("metastore") + || option.equals("token") + || option.startsWith("token.") + || option.startsWith("dlf.")) { + throw new IllegalArgumentException( + "REST catalog option is not allowed in Hadoop storage configuration: " + key); + } + } + + private Catalog createCatalog(Map<String, String> optionsMap) { + final Options options = Options.fromMap(optionsMap); final CatalogContext catalogContext = CatalogContext.create(options, paimonHadoopConfiguration); try { diff --git a/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/config/PaimonBaseOptions.java b/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/config/PaimonBaseOptions.java index 05dabf9f83..eecc3021b7 100644 --- a/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/config/PaimonBaseOptions.java +++ b/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/config/PaimonBaseOptions.java @@ -21,6 +21,8 @@ import org.apache.seatunnel.api.configuration.Option; import org.apache.seatunnel.api.configuration.Options; import org.apache.seatunnel.connectors.seatunnel.paimon.catalog.PaimonCatalogEnum; +import org.apache.paimon.rest.RESTCatalogOptions; + import java.util.HashMap; import java.util.Map; @@ -32,7 +34,8 @@ public class PaimonBaseOptions { Options.key("warehouse") .stringType() .noDefaultValue() - .withDescription("The warehouse path of paimon"); + .withDescription( + "The warehouse path of paimon, or warehouse name for REST catalogs"); public static final Option<PaimonCatalogEnum> CATALOG_TYPE = Options.key("catalog_type") @@ -44,7 +47,43 @@ public class PaimonBaseOptions { Options.key("catalog_uri") .stringType() .noDefaultValue() - .withDescription("The uri of paimon with hive catalog"); + .withDescription("The uri of paimon with Hive or REST catalog"); + + public static final Option<String> TOKEN_PROVIDER = + Options.key(RESTCatalogOptions.TOKEN_PROVIDER.key()) + .stringType() + .noDefaultValue() + .withDescription("REST catalog authentication provider: bear or dlf"); + + public static final Option<String> TOKEN = + Options.key(RESTCatalogOptions.TOKEN.key()) + .stringType() + .noDefaultValue() + .withDescription("REST catalog bearer token"); + + public static final Option<String> DLF_ACCESS_KEY_ID = + Options.key(RESTCatalogOptions.DLF_ACCESS_KEY_ID.key()) + .stringType() + .noDefaultValue() + .withDescription("DLF access key ID"); + + public static final Option<String> DLF_ACCESS_KEY_SECRET = + Options.key(RESTCatalogOptions.DLF_ACCESS_KEY_SECRET.key()) + .stringType() + .noDefaultValue() + .withDescription("DLF access key secret"); + + public static final Option<String> DLF_SECURITY_TOKEN = + Options.key(RESTCatalogOptions.DLF_SECURITY_TOKEN.key()) + .stringType() + .noDefaultValue() + .withDescription("Optional DLF temporary security token"); + + public static final Option<String> DLF_REGION = + Options.key(RESTCatalogOptions.DLF_REGION.key()) + .stringType() + .noDefaultValue() + .withDescription("DLF region; inferred from catalog_uri when omitted"); public static final Option<String> CATALOG_NAME = Options.key("catalog_name") diff --git a/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/config/PaimonConfig.java b/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/config/PaimonConfig.java index 9566cb583a..4c841967b0 100644 --- a/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/config/PaimonConfig.java +++ b/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/config/PaimonConfig.java @@ -18,8 +18,10 @@ package org.apache.seatunnel.connectors.seatunnel.paimon.config; import org.apache.seatunnel.shade.com.google.common.collect.ImmutableList; +import org.apache.seatunnel.shade.com.google.common.net.InetAddresses; import org.apache.seatunnel.shade.org.apache.commons.lang3.StringUtils; +import org.apache.seatunnel.api.configuration.Option; import org.apache.seatunnel.api.configuration.ReadonlyConfig; import org.apache.seatunnel.api.sink.SeaTunnelSink; import org.apache.seatunnel.api.source.SeaTunnelSource; @@ -28,9 +30,13 @@ import org.apache.seatunnel.common.utils.SeaTunnelException; import org.apache.seatunnel.connectors.seatunnel.paimon.catalog.PaimonCatalogEnum; import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import java.io.Serializable; +import java.net.URI; +import java.net.URISyntaxException; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -41,6 +47,7 @@ import static java.util.stream.Collectors.toList; * SeaTunnelSink}. */ @Getter +@Slf4j public class PaimonConfig implements Serializable { protected String catalogName; @@ -54,6 +61,7 @@ public class PaimonConfig implements Serializable { protected String hadoopConfPath; protected String user; protected String password; + private final Map<String, String> restOptions = new HashMap<>(); public PaimonConfig(ReadonlyConfig readonlyConfig) { this.catalogName = @@ -70,16 +78,97 @@ public class PaimonConfig implements Serializable { this.hadoopConfProps = readonlyConfig.get(PaimonBaseOptions.HADOOP_CONF); this.hadoopConfPath = readonlyConfig.get(PaimonBaseOptions.HADOOP_CONF_PATH); this.catalogType = readonlyConfig.get(PaimonBaseOptions.CATALOG_TYPE); - if (PaimonCatalogEnum.HIVE.getType().equals(catalogType.getType())) { + if (catalogType == PaimonCatalogEnum.HIVE || catalogType == PaimonCatalogEnum.REST) { this.catalogUri = checkArgumentNotBlank( readonlyConfig.get(PaimonBaseOptions.CATALOG_URI), PaimonBaseOptions.CATALOG_URI.key()); } + if (catalogType == PaimonCatalogEnum.REST) { + validateRestUri(); + String provider = readonlyConfig.get(PaimonBaseOptions.TOKEN_PROVIDER); + if (!"bear".equals(provider) && !"dlf".equals(provider)) { + throw new SeaTunnelException("Invalid token.provider: expected bear or dlf"); + } + if ("bear".equals(provider)) { + checkArgumentNotBlank( + readonlyConfig.get(PaimonBaseOptions.TOKEN), PaimonBaseOptions.TOKEN.key()); + } else { + checkArgumentNotBlank( + readonlyConfig.get(PaimonBaseOptions.DLF_ACCESS_KEY_ID), + PaimonBaseOptions.DLF_ACCESS_KEY_ID.key()); + checkArgumentNotBlank( + readonlyConfig.get(PaimonBaseOptions.DLF_ACCESS_KEY_SECRET), + PaimonBaseOptions.DLF_ACCESS_KEY_SECRET.key()); + } + for (Option<String> option : + Arrays.asList( + PaimonBaseOptions.TOKEN_PROVIDER, PaimonBaseOptions.TOKEN, + PaimonBaseOptions.DLF_ACCESS_KEY_ID, + PaimonBaseOptions.DLF_ACCESS_KEY_SECRET, + PaimonBaseOptions.DLF_SECURITY_TOKEN, PaimonBaseOptions.DLF_REGION)) { + String value = readonlyConfig.get(option); + if (value != null) { + if (option == PaimonBaseOptions.DLF_SECURITY_TOKEN) { + validateSecurityToken(value); + } + restOptions.put(option.key(), value); + } + } + } this.user = readonlyConfig.get(PaimonBaseOptions.USER); this.password = readonlyConfig.get(PaimonBaseOptions.PASSWORD); } + private void validateRestUri() { + URI uri; + try { + uri = new URI(catalogUri); + } catch (URISyntaxException e) { + // URI syntax errors include the input, which may contain credentials. + throw invalidRestUri(); + } + if (!("http".equalsIgnoreCase(uri.getScheme()) || "https".equalsIgnoreCase(uri.getScheme())) + || StringUtils.isBlank(uri.getHost()) + || uri.getRawUserInfo() != null) { + throw invalidRestUri(); + } + if ("http".equalsIgnoreCase(uri.getScheme()) && !isLoopback(uri.getHost())) { + log.warn( + "catalog_uri uses HTTP outside loopback; REST credentials will be sent in clear text. Use HTTPS."); + } + } + + private SeaTunnelException invalidRestUri() { + return new SeaTunnelException( + "Invalid catalog_uri: REST requires HTTP or HTTPS, a host, and no user information"); + } + + private boolean isLoopback(String host) { + if ("localhost".equalsIgnoreCase(host)) { + return true; + } + String address = + host.startsWith("[") && host.endsWith("]") + ? host.substring(1, host.length() - 1) + : host; + // Do not resolve DNS names during configuration validation. + return InetAddresses.isInetAddress(address) + && InetAddresses.forString(address).isLoopbackAddress(); + } + + private void validateSecurityToken(String value) { + for (int i = 0; i < value.length(); i++) { + char character = value.charAt(i); + if (character != '\t' && (character < ' ' || character >= 127)) { + // OkHttp includes invalid non-Authorization header values in its exception text. + throw new SeaTunnelException( + "Invalid HTTP header characters in " + + PaimonBaseOptions.DLF_SECURITY_TOKEN.key()); + } + } + } + protected String checkArgumentNotBlank(String propValue, String propKey) { if (StringUtils.isBlank(propValue)) { throw new SeaTunnelException( diff --git a/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/sink/PaimonSinkFactory.java b/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/sink/PaimonSinkFactory.java index d578c48435..e9fb783ffe 100644 --- a/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/sink/PaimonSinkFactory.java +++ b/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/sink/PaimonSinkFactory.java @@ -58,6 +58,12 @@ public class PaimonSinkFactory implements TableSinkFactory { PaimonSinkOptions.HADOOP_CONF, PaimonSinkOptions.HADOOP_CONF_PATH, PaimonSinkOptions.CATALOG_TYPE, + PaimonSinkOptions.TOKEN_PROVIDER, + PaimonSinkOptions.TOKEN, + PaimonSinkOptions.DLF_ACCESS_KEY_ID, + PaimonSinkOptions.DLF_ACCESS_KEY_SECRET, + PaimonSinkOptions.DLF_SECURITY_TOKEN, + PaimonSinkOptions.DLF_REGION, PaimonSinkOptions.USER, PaimonSinkOptions.PASSWORD, PaimonSinkOptions.SCHEMA_SAVE_MODE, @@ -77,6 +83,11 @@ public class PaimonSinkFactory implements TableSinkFactory { PaimonSinkOptions.CATALOG_TYPE, PaimonCatalogEnum.HIVE, PaimonSinkOptions.CATALOG_URI) + .conditional( + PaimonSinkOptions.CATALOG_TYPE, + PaimonCatalogEnum.REST, + PaimonSinkOptions.CATALOG_URI, + PaimonSinkOptions.TOKEN_PROVIDER) .build(); } diff --git a/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/source/PaimonSourceFactory.java b/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/source/PaimonSourceFactory.java index 60d9da2a46..c33bb1001b 100644 --- a/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/source/PaimonSourceFactory.java +++ b/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/source/PaimonSourceFactory.java @@ -59,6 +59,12 @@ public class PaimonSourceFactory implements TableSourceFactory { PaimonSourceOptions.QUERY_SQL, PaimonSourceOptions.HADOOP_CONF, PaimonSourceOptions.HADOOP_CONF_PATH, + PaimonSourceOptions.TOKEN_PROVIDER, + PaimonSourceOptions.TOKEN, + PaimonSourceOptions.DLF_ACCESS_KEY_ID, + PaimonSourceOptions.DLF_ACCESS_KEY_SECRET, + PaimonSourceOptions.DLF_SECURITY_TOKEN, + PaimonSourceOptions.DLF_REGION, PaimonSourceOptions.USER, PaimonSourceOptions.PASSWORD) .exclusive(PaimonSourceOptions.TABLE, CatalogOptions.TABLE_LIST) @@ -66,6 +72,11 @@ public class PaimonSourceFactory implements TableSourceFactory { PaimonSourceOptions.CATALOG_TYPE, PaimonCatalogEnum.HIVE, PaimonSourceOptions.CATALOG_URI) + .conditional( + PaimonSourceOptions.CATALOG_TYPE, + PaimonCatalogEnum.REST, + PaimonSourceOptions.CATALOG_URI, + PaimonSourceOptions.TOKEN_PROVIDER) .build(); } diff --git a/seatunnel-connectors-v2/connector-paimon/src/test/java/org/apache/seatunnel/connectors/seatunnel/paimon/catalog/PaimonCatalogLoaderTest.java b/seatunnel-connectors-v2/connector-paimon/src/test/java/org/apache/seatunnel/connectors/seatunnel/paimon/catalog/PaimonCatalogLoaderTest.java new file mode 100644 index 0000000000..376433aa0d --- /dev/null +++ b/seatunnel-connectors-v2/connector-paimon/src/test/java/org/apache/seatunnel/connectors/seatunnel/paimon/catalog/PaimonCatalogLoaderTest.java @@ -0,0 +1,505 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.paimon.catalog; + +import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.common.utils.SeaTunnelException; +import org.apache.seatunnel.connectors.seatunnel.paimon.config.PaimonBaseOptions; +import org.apache.seatunnel.connectors.seatunnel.paimon.config.PaimonConfig; +import org.apache.seatunnel.connectors.seatunnel.paimon.security.PaimonSecurityContext; + +import org.apache.hadoop.conf.Configuration; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.Appender; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.Logger; +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.catalog.CatalogFactory; +import org.apache.paimon.options.Options; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.ArgumentCaptor; +import org.mockito.MockedStatic; + +import java.io.Writer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class PaimonCatalogLoaderTest { + @TempDir private Path directory; + + @Test + void testRestForwardsOnlyExplicitStorageOptions() { + Map<String, Object> config = config("rest"); + Map<String, String> storage = new HashMap<>(); + storage.put("fs.s3a.endpoint", "storage.example.com"); + storage.put("dfs.nameservices", "managed-storage"); + config.put("paimon.hadoop.conf", storage); + assertEquals(expectedRestOptions(storage), catalogOptions(config)); + // A managed warehouse is an opaque name, not an instruction to load Hadoop defaults. + config.put("warehouse", "s3a://managed-warehouse"); + Map<String, String> expected = expectedRestOptions(storage); + expected.put("warehouse", "s3a://managed-warehouse"); + assertEquals(expected, catalogOptions(config)); + } + + @Test + void testRestRejectsCatalogAndAuthOptionsInStorageMap() { + for (String key : + new String[] { + "uri", + "warehouse", + "metastore", + "token", + "token.provider", + "token.refresh-time", + "dlf.token-path", + "dlf.token-loader", + "dlf.token-ecs-endpoint", + "dlf.security-token", + "hadoop.uri", + "hadoop.token", + "hadoop.dlf.token-path", + "hadoop.hadoop.dlf.token-loader" + }) { + Map<String, Object> config = config("rest"); + config.put( + "paimon.hadoop.conf", + Collections.singletonMap(key, "private-value\nsecret-tail")); + RuntimeException error = + assertThrows(RuntimeException.class, () -> catalogOptions(config), key); + assertTrue(error.getMessage().contains(key)); + assertFalse(error.getMessage().contains("private-value")); + assertFalse(error.getMessage().contains("secret-tail")); + assertNull(error.getCause()); + } + } + + @Test + void testRestRejectsCatalogAndAuthOptionsInStorageFiles() throws Exception { + for (String option : + new String[] {"hdfs_site_path", PaimonBaseOptions.HADOOP_CONF_PATH.key()}) { + for (String key : + new String[] {"dlf.token-path", "token", "uri", "hadoop.dlf.security-token"}) { + Path file = directory.resolve("hdfs-site.xml"); + writeConfiguration(file, Collections.singletonMap(key, "private-file-value")); + Map<String, Object> config = config("rest"); + config.put( + option, + option.equals("hdfs_site_path") ? file.toString() : directory.toString()); + RuntimeException error = + assertThrows(RuntimeException.class, () -> catalogOptions(config), key); + assertTrue(error.getMessage().contains(key)); + assertFalse(error.getMessage().contains("private-file-value")); + assertNull(error.getCause()); + } + } + } + + @Test + void testRestLoadsOnlyExplicitFileOptionsWithMapPrecedence() throws Exception { + Path site = directory.resolve("custom-site.xml"); + writeConfiguration(site, Collections.singletonMap("dfs.nameservices", "site-storage")); + Map<String, String> fileOptions = new HashMap<>(); + fileOptions.put("fs.s3a.endpoint", "file-endpoint"); + fileOptions.put("fs.s3a.path.style.access", "true"); + writeConfiguration(directory.resolve("core-site.xml"), fileOptions); + Map<String, Object> config = config("rest"); + config.put("hdfs_site_path", site.toString()); + config.put(PaimonBaseOptions.HADOOP_CONF_PATH.key(), directory.toString()); + config.put( + "paimon.hadoop.conf", Collections.singletonMap("fs.s3a.endpoint", "map-endpoint")); + fileOptions.put("fs.s3a.endpoint", "map-endpoint"); + fileOptions.put("dfs.nameservices", "site-storage"); + assertEquals(expectedRestOptions(fileOptions), catalogOptions(config)); + } + + @Test + void testRestRejectsUnsafeUrisWithoutEchoingValues() { + for (String uri : + new String[] { + "127.0.0.1:1", "https://user:[email protected]", + "https:///path", + "ftp://catalog.example.com", "https://", "https://bad host/private-value" + }) { + Map<String, Object> config = config("rest"); + config.put("catalog_uri", uri); + SeaTunnelException error = + assertThrows( + SeaTunnelException.class, + () -> new PaimonConfig(ReadonlyConfig.fromMap(config))); + assertTrue(error.getMessage().contains("catalog_uri")); + assertFalse(error.getMessage().contains(uri)); + assertFalse(error.getMessage().contains("private-password")); + assertNull(error.getCause()); + } + } + + @Test + void testRestUriWarnsOnlyForNonLoopbackHttpWithoutEchoingValues() { + Logger logger = (Logger) LogManager.getLogger(PaimonConfig.class); + Level previous = logger.getLevel(); + Appender appender = mock(Appender.class); + when(appender.getName()).thenReturn("rest-uri-test"); + when(appender.isStarted()).thenReturn(true); + logger.addAppender(appender); + logger.setLevel(Level.WARN); + try { + for (String uri : + new String[] { + "https://catalog.example.com", + "http://localhost:8080", + "http://127.0.0.1:8080", + "http://[::1]:8080" + }) { + Map<String, Object> config = config("rest"); + config.put("catalog_uri", uri); + new PaimonConfig(ReadonlyConfig.fromMap(config)); + } + verify(appender, times(0)).append(any(LogEvent.class)); + Map<String, Object> config = config("rest"); + config.put("catalog_uri", "http://remote-catalog.example.com"); + new PaimonConfig(ReadonlyConfig.fromMap(config)); + ArgumentCaptor<LogEvent> event = ArgumentCaptor.forClass(LogEvent.class); + verify(appender).append(event.capture()); + assertEquals(Level.WARN, event.getValue().getLevel()); + assertTrue(event.getValue().getMessage().getFormattedMessage().contains("catalog_uri")); + assertFalse( + event.getValue().getMessage().getFormattedMessage().contains("remote-catalog")); + } finally { + logger.removeAppender(appender); + logger.setLevel(previous); + } + } + + @Test + void testRestRequiresSupportedExactTokenProvider() { + for (String provider : new String[] {null, "", " ", "BEAR", "private-provider"}) { + Map<String, Object> config = config("rest"); + config.put("token.provider", provider); + SeaTunnelException error = + assertThrows( + SeaTunnelException.class, + () -> new PaimonConfig(ReadonlyConfig.fromMap(config))); + assertTrue(error.getMessage().contains("token.provider")); + assertTrue(error.getMessage().contains("bear")); + assertTrue(error.getMessage().contains("dlf")); + assertFalse(error.getMessage().contains("private-provider")); + assertNull(error.getCause()); + } + } + + @Test + void testRestBearerRequiresNonBlankToken() { + for (String token : new String[] {null, "", " "}) { + Map<String, Object> config = config("rest"); + config.put("token", token); + SeaTunnelException error = + assertThrows( + SeaTunnelException.class, + () -> new PaimonConfig(ReadonlyConfig.fromMap(config))); + assertTrue(error.getMessage().contains("token")); + assertFalse(error.toString().contains("test-bearer")); + } + } + + @Test + void testRestDlfRequiresBothAccessKeys() { + for (String key : new String[] {"dlf.access-key-id", "dlf.access-key-secret"}) { + for (String value : new String[] {null, "", " "}) { + Map<String, Object> config = config("rest"); + config.put("token.provider", "dlf"); + config.put("dlf.access-key-id", "test-id"); + config.put("dlf.access-key-secret", "test-secret"); + config.put(key, value); + SeaTunnelException error = + assertThrows( + SeaTunnelException.class, + () -> new PaimonConfig(ReadonlyConfig.fromMap(config))); + assertTrue(error.getMessage().contains(key)); + assertFalse(error.getMessage().contains("test-secret")); + assertFalse(String.valueOf(error.getCause()).contains("test-secret")); + } + } + } + + @Test + void testRestBearerOptions() { + Map<String, Object> config = config("rest"); + config.put("token.provider", "bear"); + config.put("token", "test-bearer"); + Map<String, String> options = catalogOptions(config); + assertEquals("rest", options.get("metastore")); + assertEquals("http://localhost:8080", options.get("uri")); + assertEquals("managed-warehouse", options.get("warehouse")); + assertEquals("bear", options.get("token.provider")); + assertEquals("test-bearer", options.get("token")); + } + + @Test + void testQuotedHoconRestOptions() { + ReadonlyConfig config = + ReadonlyConfig.fromConfig( + ConfigFactory.parseString( + "warehouse=managed-warehouse\ncatalog_type=rest\ncatalog_uri=\"https://catalog.example.com\"\n" + + "\"token.provider\"=bear\ntoken=test-bearer\n" + + "\"dlf.access-key-id\"=test-id\n\"dlf.access-key-secret\"=test-secret")); + Map<String, String> options = new PaimonConfig(config).getRestOptions(); + assertEquals("bear", options.get("token.provider")); + assertEquals("test-bearer", options.get("token")); + assertEquals("test-secret", options.get("dlf.access-key-secret")); + } + + @Test + void testRestDlfOptions() { + Map<String, Object> config = config("rest"); + config.put("token.provider", "dlf"); + config.put("dlf.access-key-id", "test-id"); + config.put("dlf.access-key-secret", "test-secret"); + config.put("dlf.security-token", "test-session"); + config.put("dlf.region", "cn-hangzhou"); + Map<String, String> options = catalogOptions(config); + for (String key : + new String[] { + "token.provider", + "dlf.access-key-id", + "dlf.access-key-secret", + "dlf.security-token", + "dlf.region" + }) { + assertEquals(config.get(key), options.get(key)); + } + } + + @Test + void testRestDoesNotForwardAnUnconfiguredRegion() { + Map<String, Object> config = config("rest"); + config.put("catalog_uri", "https://cn-hangzhou.example.com"); + config.put("token.provider", "dlf"); + config.put("dlf.access-key-id", "test-id"); + config.put("dlf.access-key-secret", "test-secret"); + assertFalse(catalogOptions(config).containsKey("dlf.region")); + } + + @Test + void testRestRetainsHadoopStorageConfiguration() { + Map<String, Object> config = config("rest"); + Map<String, String> hadoop = new HashMap<>(); + hadoop.put("dfs.nameservices", "managed-storage"); + hadoop.put("dfs.ha.namenodes.managed-storage", "nn1,nn2"); + hadoop.put("fs.s3a.endpoint", "storage.example.com"); + hadoop.put("fs.hdfs.impl.disable.cache", "true"); + config.put("paimon.hadoop.conf", hadoop); + Map<String, String> options = catalogOptions(config); + // RESTCatalog rebuilds its context after merging the server's configuration. + CatalogContext effective = CatalogContext.create(Options.fromMap(options)); + assertEquals("managed-storage", effective.hadoopConf().get("dfs.nameservices")); + assertEquals("nn1,nn2", effective.hadoopConf().get("dfs.ha.namenodes.managed-storage")); + assertEquals("storage.example.com", options.get("fs.s3a.endpoint")); + assertEquals("true", effective.hadoopConf().get("fs.hdfs.impl.disable.cache")); + assertEquals("managed-warehouse", options.get("warehouse")); + assertEquals("rest", options.get("metastore")); + assertEquals("http://localhost:8080", options.get("uri")); + } + + @Test + void testMalformedDlfSecurityTokenIsRejectedWithoutLeakingIt() { + for (String token : + new String[] { + "test-session\ncontinuation", + "test-session\rcontinuation", + "test-session\u0000continuation", + "test-session\u0080continuation" + }) { + Map<String, Object> config = config("rest"); + config.put("token.provider", "dlf"); + config.put("dlf.access-key-id", "test-id"); + config.put("dlf.access-key-secret", "test-secret"); + config.put("dlf.security-token", token); + SeaTunnelException error = + assertThrows( + SeaTunnelException.class, + () -> new PaimonConfig(ReadonlyConfig.fromMap(config))); + assertTrue(error.getMessage().contains("dlf.security-token")); + for (Throwable cause = error; cause != null; cause = cause.getCause()) { + assertFalse(String.valueOf(cause.getMessage()).contains("test-session")); + assertFalse(String.valueOf(cause.getMessage()).contains("continuation")); + } + } + } + + @Test + void testRestRetainsSecurityConfigurationAfterInitialization() { + try (MockedStatic<PaimonSecurityContext> security = + mockStatic(PaimonSecurityContext.class, CALLS_REAL_METHODS)) { + security.when( + () -> + PaimonSecurityContext.shouldEnableKerberos( + any(Configuration.class))) + .thenAnswer( + invocation -> { + Configuration configuration = invocation.getArgument(0); + configuration.set("hadoop.security.authentication", "kerberos"); + return null; + }); + CatalogContext effective = + CatalogContext.create(Options.fromMap(catalogOptions(config("rest")))); + assertEquals("kerberos", effective.hadoopConf().get("hadoop.security.authentication")); + } + } + + @Test + void testRestMissingOrBlankUriFailsWithConnectorError() { + for (String uri : new String[] {null, "", " "}) { + Map<String, Object> config = config("rest"); + config.put("catalog_uri", uri); + SeaTunnelException error = + assertThrows( + SeaTunnelException.class, + () -> new PaimonConfig(ReadonlyConfig.fromMap(config))); + assertTrue(error.getMessage().contains("catalog_uri")); + } + } + + @Test + void testFilesystemOptionsUnchanged() { + Map<String, Object> config = config("filesystem"); + config.put("warehouse", "file:///tmp/paimon-test"); + config.put("user", "test-user"); + config.put("password", "test-password"); + config.put("token", "ignored"); + Map<String, String> expected = new HashMap<>(); + expected.put("metastore", "filesystem"); + expected.put("warehouse", "file:///tmp/paimon-test"); + expected.put("user", "test-user"); + expected.put("password", "test-password"); + assertEquals(expected, catalogOptions(config)); + } + + @Test + void testHiveOptionsUnchanged() { + Map<String, Object> config = config("hive"); + config.put("warehouse", "file:///tmp/paimon-hive"); + config.put("catalog_uri", "thrift://localhost:9083"); + Map<String, String> expected = + new HashMap<>( + PaimonSecurityContext.loadHadoopConfig( + new PaimonConfig(ReadonlyConfig.fromMap(config))) + .getPropsWithPrefix("")); + expected.put("metastore", "hive"); + expected.put("warehouse", "file:///tmp/paimon-hive"); + expected.put("uri", "thrift://localhost:9083"); + assertEquals(expected, catalogOptions(config)); + } + + @Test + void testRestCatalogIsClosed() throws Exception { + Catalog delegate = mock(Catalog.class); + ClassLoader previous = Thread.currentThread().getContextClassLoader(); + try (MockedStatic<CatalogFactory> factory = mockStatic(CatalogFactory.class)) { + factory.when(() -> CatalogFactory.createCatalog(any(CatalogContext.class))) + .thenReturn(delegate); + try (PaimonCatalog catalog = + new PaimonCatalog("test", ReadonlyConfig.fromMap(config("rest")))) { + catalog.open(); + } + verify(delegate).close(); + } finally { + Thread.currentThread().setContextClassLoader(previous); + } + } + + @Test + void testUnknownCatalogTypeStillFails() { + Map<String, Object> config = config("unknown"); + assertThrows( + IllegalArgumentException.class, + () -> new PaimonConfig(ReadonlyConfig.fromMap(config))); + } + + private Map<String, Object> config(String type) { + Map<String, Object> config = new HashMap<>(); + config.put("catalog_type", type); + config.put("catalog_uri", "http://localhost:8080"); + config.put("warehouse", "managed-warehouse"); + if (type.equals("rest")) { + config.put("token.provider", "bear"); + config.put("token", "test-bearer"); + } + return config; + } + + private Map<String, String> expectedRestOptions(Map<String, String> storage) { + Map<String, String> expected = new HashMap<>(storage); + storage.forEach((key, value) -> expected.put("hadoop." + key, value)); + expected.put("uri", "http://localhost:8080"); + expected.put("warehouse", "managed-warehouse"); + expected.put("metastore", "rest"); + expected.put("token.provider", "bear"); + expected.put("token", "test-bearer"); + return expected; + } + + private void writeConfiguration(Path file, Map<String, String> values) throws Exception { + Configuration configuration = new Configuration(false); + values.forEach(configuration::set); + try (Writer writer = Files.newBufferedWriter(file)) { + configuration.writeXml(writer); + } + } + + private Map<String, String> catalogOptions(Map<String, Object> config) { + AtomicReference<Map<String, String>> options = new AtomicReference<>(); + ClassLoader previous = Thread.currentThread().getContextClassLoader(); + try (MockedStatic<CatalogFactory> factory = mockStatic(CatalogFactory.class)) { + factory.when(() -> CatalogFactory.createCatalog(any(CatalogContext.class))) + .thenAnswer( + invocation -> { + CatalogContext context = invocation.getArgument(0); + options.set(new HashMap<>(context.options().toMap())); + return mock(Catalog.class); + }); + new PaimonCatalogLoader(new PaimonConfig(ReadonlyConfig.fromMap(config))).loadCatalog(); + return options.get(); + } finally { + Thread.currentThread().setContextClassLoader(previous); + } + } +} diff --git a/seatunnel-connectors-v2/connector-paimon/src/test/java/org/apache/seatunnel/connectors/seatunnel/paimon/config/PaimonOptionRuleTest.java b/seatunnel-connectors-v2/connector-paimon/src/test/java/org/apache/seatunnel/connectors/seatunnel/paimon/config/PaimonOptionRuleTest.java index 92e159fcb7..675346885c 100644 --- a/seatunnel-connectors-v2/connector-paimon/src/test/java/org/apache/seatunnel/connectors/seatunnel/paimon/config/PaimonOptionRuleTest.java +++ b/seatunnel-connectors-v2/connector-paimon/src/test/java/org/apache/seatunnel/connectors/seatunnel/paimon/config/PaimonOptionRuleTest.java @@ -18,7 +18,10 @@ package org.apache.seatunnel.connectors.seatunnel.paimon.config; import org.apache.seatunnel.api.configuration.Option; +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConfigValidator; import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.OptionValidationException; import org.apache.seatunnel.api.options.SinkConnectorCommonOptions; import org.apache.seatunnel.connectors.seatunnel.paimon.catalog.PaimonCatalogFactory; import org.apache.seatunnel.connectors.seatunnel.paimon.sink.PaimonSinkFactory; @@ -27,14 +30,52 @@ import org.apache.seatunnel.connectors.seatunnel.paimon.source.PaimonSourceFacto import org.junit.jupiter.api.Test; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.stream.Stream; import static java.util.stream.Collectors.toList; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; class PaimonOptionRuleTest { + @Test + void testRestOptionsDeclaredByAllFactories() { + for (OptionRule rule : + Arrays.asList( + new PaimonSourceFactory().optionRule(), + new PaimonSinkFactory().optionRule(), + new PaimonCatalogFactory().optionRule())) { + assertDeclaredOptions( + rule, + PaimonBaseOptions.TOKEN_PROVIDER, + PaimonBaseOptions.TOKEN, + PaimonBaseOptions.DLF_ACCESS_KEY_ID, + PaimonBaseOptions.DLF_ACCESS_KEY_SECRET, + PaimonBaseOptions.DLF_SECURITY_TOKEN, + PaimonBaseOptions.DLF_REGION); + Map<String, Object> options = new HashMap<>(); + options.put("catalog_type", "rest"); + options.put("warehouse", "managed-warehouse"); + options.put("database", "test_db"); + options.put("table", "test_table"); + assertThrows( + OptionValidationException.class, + () -> ConfigValidator.of(ReadonlyConfig.fromMap(options)).validate(rule)); + options.put("catalog_uri", "https://catalog.example.com"); + assertThrows( + OptionValidationException.class, + () -> ConfigValidator.of(ReadonlyConfig.fromMap(options)).validate(rule)); + options.put("token.provider", "bear"); + options.put("token", "test-bearer"); + assertDoesNotThrow( + () -> ConfigValidator.of(ReadonlyConfig.fromMap(options)).validate(rule)); + } + } + @Test void testSinkOptionRuleDeclaresNonPrimaryKey() { assertDeclaredOptions( diff --git a/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigShadeUtils.java b/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigShadeUtils.java index 714b7158c0..2faadc0988 100644 --- a/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigShadeUtils.java +++ b/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigShadeUtils.java @@ -62,6 +62,8 @@ public final class ConfigShadeUtils { "client_secret", "security_token", "connection_string", + "dlf.access-key-id", + "dlf.access-key-secret", "consumer_key", "consumer_secret" }; diff --git a/seatunnel-core/seatunnel-core-starter/src/test/java/org/apache/seatunnel/core/starter/utils/ConfigBuilderTest.java b/seatunnel-core/seatunnel-core-starter/src/test/java/org/apache/seatunnel/core/starter/utils/ConfigBuilderTest.java index 126b955e49..51588da648 100644 --- a/seatunnel-core/seatunnel-core-starter/src/test/java/org/apache/seatunnel/core/starter/utils/ConfigBuilderTest.java +++ b/seatunnel-core/seatunnel-core-starter/src/test/java/org/apache/seatunnel/core/starter/utils/ConfigBuilderTest.java @@ -303,6 +303,42 @@ public class ConfigBuilderTest { ConfigShadeUtils.getSensitiveOptions(null).contains("security_token")); } + @Test + public void testPaimonRestCredentialsAreMaskedWithoutChangingEncryptionKeys() { + Map<String, Object> connector = new LinkedHashMap<>(); + connector.put("token", "test-bearer"); + connector.put("token.provider", "bear"); + connector.put("dlf.access-key-id", "test-id"); + connector.put("dlf.access-key-secret", "test-secret"); + connector.put("dlf.security-token", "test-session"); + Map<String, Object> dlf = new LinkedHashMap<>(); + dlf.put("access-key-id", "nested-id"); + dlf.put("access-key-secret", "nested-secret"); + dlf.put("security-token", "nested-session"); + dlf.put("region", "cn-hangzhou"); + connector.put("dlf", dlf); + Map<String, Object> masked = + ConfigBuilder.configDesensitization( + connector, ConfigShadeUtils.getLogDesensitizationOptions(null)); + for (String key : + Arrays.asList( + "token", + "dlf.access-key-id", + "dlf.access-key-secret", + "dlf.security-token")) { + Assertions.assertEquals("******", masked.get(key)); + } + Assertions.assertEquals("bear", masked.get("token.provider")); + Map<?, ?> maskedDlf = (Map<?, ?>) masked.get("dlf"); + for (String key : Arrays.asList("access-key-id", "access-key-secret", "security-token")) { + Assertions.assertEquals("******", maskedDlf.get(key)); + } + Assertions.assertEquals("cn-hangzhou", maskedDlf.get("region")); + Assertions.assertEquals("test-secret", connector.get("dlf.access-key-secret")); + Assertions.assertFalse( + ConfigShadeUtils.getSensitiveOptions(null).contains("dlf.access-key-secret")); + } + @Test public void testConfigDesensitizationMasksWooCommerceCredentialsWithoutAddingEncryptionKeys() { Map<String, Object> connector = new LinkedHashMap<>(); diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-paimon-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-paimon-e2e/pom.xml index bc1661947f..6e3f272820 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-paimon-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-paimon-e2e/pom.xml @@ -28,6 +28,7 @@ <properties> <libfb303.version>0.9.0</libfb303.version> <minio.version>8.5.6</minio.version> + <paimon.version>1.1.1</paimon.version> </properties> <dependencyManagement> @@ -43,6 +44,30 @@ </dependencyManagement> <dependencies> + <dependency> + <groupId>org.apache.paimon</groupId> + <artifactId>paimon-core</artifactId> + <version>${paimon.version}</version> + <classifier>tests</classifier> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>*</groupId> + <artifactId>*</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>com.squareup.okhttp3</groupId> + <artifactId>mockwebserver</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.seatunnel</groupId> + <artifactId>connector-cdc-base</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> <!-- minio containers --> <dependency> <groupId>org.testcontainers</groupId> diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-paimon-e2e/src/test/java/org/apache/seatunnel/e2e/connector/paimon/PaimonRestCatalogIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-paimon-e2e/src/test/java/org/apache/seatunnel/e2e/connector/paimon/PaimonRestCatalogIT.java new file mode 100644 index 0000000000..d646174aff --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-paimon-e2e/src/test/java/org/apache/seatunnel/e2e/connector/paimon/PaimonRestCatalogIT.java @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.e2e.connector.paimon; + +import org.apache.seatunnel.api.common.JobContext; +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.sink.DefaultSinkWriterContext; +import org.apache.seatunnel.api.sink.SaveModeHandler; +import org.apache.seatunnel.api.sink.SinkAggregatedCommitter; +import org.apache.seatunnel.api.source.Boundedness; +import org.apache.seatunnel.api.source.Collector; +import org.apache.seatunnel.api.source.SourceReader; +import org.apache.seatunnel.api.source.SourceSplitEnumerator; +import org.apache.seatunnel.api.table.catalog.CatalogTable; +import org.apache.seatunnel.api.table.catalog.PhysicalColumn; +import org.apache.seatunnel.api.table.catalog.PrimaryKey; +import org.apache.seatunnel.api.table.catalog.TableIdentifier; +import org.apache.seatunnel.api.table.catalog.TablePath; +import org.apache.seatunnel.api.table.catalog.TableSchema; +import org.apache.seatunnel.api.table.factory.TableSinkFactoryContext; +import org.apache.seatunnel.api.table.factory.TableSourceFactoryContext; +import org.apache.seatunnel.api.table.type.BasicType; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.common.constants.JobMode; +import org.apache.seatunnel.connectors.seatunnel.paimon.catalog.PaimonCatalog; +import org.apache.seatunnel.connectors.seatunnel.paimon.catalog.PaimonCatalogFactory; +import org.apache.seatunnel.connectors.seatunnel.paimon.catalog.PaimonCatalogLoader; +import org.apache.seatunnel.connectors.seatunnel.paimon.config.PaimonConfig; +import org.apache.seatunnel.connectors.seatunnel.paimon.exception.PaimonConnectorException; +import org.apache.seatunnel.connectors.seatunnel.paimon.sink.PaimonSink; +import org.apache.seatunnel.connectors.seatunnel.paimon.sink.PaimonSinkFactory; +import org.apache.seatunnel.connectors.seatunnel.paimon.sink.PaimonSinkWriter; +import org.apache.seatunnel.connectors.seatunnel.paimon.sink.commit.PaimonAggregatedCommitInfo; +import org.apache.seatunnel.connectors.seatunnel.paimon.sink.commit.PaimonCommitInfo; +import org.apache.seatunnel.connectors.seatunnel.paimon.source.PaimonSource; +import org.apache.seatunnel.connectors.seatunnel.paimon.source.PaimonSourceFactory; +import org.apache.seatunnel.connectors.seatunnel.paimon.source.PaimonSourceSplit; +import org.apache.seatunnel.connectors.seatunnel.paimon.source.PaimonSourceState; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.reader.RecordReaderIterator; +import org.apache.paimon.rest.RESTCatalogInternalOptions; +import org.apache.paimon.rest.RESTCatalogServer; +import org.apache.paimon.rest.auth.BearTokenAuthProvider; +import org.apache.paimon.rest.auth.DLFAuthProvider; +import org.apache.paimon.rest.responses.ConfigResponse; +import org.apache.paimon.table.Table; +import org.apache.paimon.table.source.ReadBuilder; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@Timeout(60) +class PaimonRestCatalogIT { + @TempDir private Path directory; + + @Test + void testBearerCatalogSourceSinkAndSaveMode() throws Exception { + RESTCatalogServer server = + new RESTCatalogServer( + directory.toString(), + new BearTokenAuthProvider("test-bearer"), + serverConfig(), + "managed-warehouse"); + server.start(); + ClassLoader previous = Thread.currentThread().getContextClassLoader(); + try { + Map<String, Object> options = options(server.getUrl()); + options.put("token.provider", "bear"); + options.put("token", "test-bearer"); + ReadonlyConfig config = ReadonlyConfig.fromMap(options); + CatalogTable table = catalogTable(); + PaimonSink sink = + (PaimonSink) + new PaimonSinkFactory() + .createSink( + new TableSinkFactoryContext( + table, config, getClass().getClassLoader())) + .createSink(); + try (SaveModeHandler handler = sink.getSaveModeHandler().get()) { + handler.open(); + handler.handleSaveMode(); + } + JobContext job = new JobContext(1L); + job.setJobMode(JobMode.BATCH); + job.setEnableCheckpoint(false); + sink.setJobContext(job); + PaimonSinkWriter writer = sink.createWriter(new DefaultSinkWriterContext(0, 1)); + SinkAggregatedCommitter<PaimonCommitInfo, PaimonAggregatedCommitInfo> committer = + sink.createAggregatedCommitter().get(); + try { + writer.write(new SeaTunnelRow(new Object[] {1, "rest-row"})); + PaimonCommitInfo commit = writer.prepareCommit(1L).get(); + committer.init(); + assertTrue( + committer + .commit( + Collections.singletonList( + committer.combine( + Collections.singletonList(commit)))) + .isEmpty()); + } finally { + writer.close(); + committer.close(); + } + PaimonSource source = + (PaimonSource) + new PaimonSourceFactory() + .<SeaTunnelRow, PaimonSourceSplit, PaimonSourceState> + createSource( + new TableSourceFactoryContext( + config, getClass().getClassLoader())) + .createSource(); + assertEquals(1, source.getProducedCatalogTables().size()); + source.setJobContext(job); + readThroughSource(source); + try (PaimonCatalog catalog = + (PaimonCatalog) new PaimonCatalogFactory().createCatalog("paimon", config)) { + catalog.open(); + assertTrue(catalog.listDatabases().contains("test_db")); + assertTrue(catalog.listTables("test_db").contains("test_table")); + Table paimonTable = catalog.getPaimonTable(TablePath.of("test_db", "test_table")); + ReadBuilder read = paimonTable.newReadBuilder(); + try (RecordReaderIterator<InternalRow> rows = + new RecordReaderIterator<>( + read.newRead().createReader(read.newScan().plan()))) { + assertTrue(rows.hasNext()); + InternalRow row = rows.next(); + assertEquals(1, row.getInt(0)); + assertEquals("rest-row", row.getString(1).toString()); + assertFalse(rows.hasNext()); + } + } + options.put("token", "wrong-bearer"); + assertThrows( + PaimonConnectorException.class, + () -> + new PaimonCatalogLoader( + new PaimonConfig(ReadonlyConfig.fromMap(options))) + .loadCatalog()); + } finally { + Thread.currentThread().setContextClassLoader(previous); + server.shutdown(); + } + } + + @Test + void testServerStorageDefaultIsNotOverriddenByHadoopDefaults() throws Exception { + Map<String, String> defaults = new HashMap<>(); + defaults.put(RESTCatalogInternalOptions.PREFIX.key(), "test"); + defaults.put("fs.s3a.path.style.access", "true"); + RESTCatalogServer server = + new RESTCatalogServer( + directory.toString(), + new BearTokenAuthProvider("test-bearer"), + new ConfigResponse(defaults, Collections.emptyMap()), + "managed-warehouse"); + server.start(); + ClassLoader previous = Thread.currentThread().getContextClassLoader(); + try { + Map<String, Object> options = options(server.getUrl()); + options.put("token.provider", "bear"); + options.put("token", "test-bearer"); + try (Catalog catalog = + new PaimonCatalogLoader(new PaimonConfig(ReadonlyConfig.fromMap(options))) + .loadCatalog()) { + assertEquals("true", catalog.options().get("fs.s3a.path.style.access")); + } + } finally { + Thread.currentThread().setContextClassLoader(previous); + server.shutdown(); + } + } + + @SuppressWarnings("unchecked") + private void readThroughSource(PaimonSource source) throws Exception { + SourceReader.Context readerContext = mock(SourceReader.Context.class); + when(readerContext.getBoundedness()).thenReturn(Boundedness.BOUNDED); + SourceSplitEnumerator.Context<PaimonSourceSplit> enumeratorContext = + mock(SourceSplitEnumerator.Context.class); + when(enumeratorContext.registeredReaders()).thenReturn(Collections.singleton(0)); + when(enumeratorContext.currentParallelism()).thenReturn(1); + List<SeaTunnelRow> records = new ArrayList<>(); + try (SourceReader<SeaTunnelRow, PaimonSourceSplit> reader = + source.createReader(readerContext); + SourceSplitEnumerator<PaimonSourceSplit, PaimonSourceState> enumerator = + source.createEnumerator(enumeratorContext)) { + doAnswer( + invocation -> { + reader.addSplits(invocation.getArgument(1)); + return null; + }) + .when(enumeratorContext) + .assignSplit(eq(0), anyList()); + doAnswer( + invocation -> { + reader.handleNoMoreSplits(); + return null; + }) + .when(enumeratorContext) + .signalNoMoreSplits(0); + reader.open(); + enumerator.open(); + enumerator.registerReader(0); + enumerator.run(); + reader.pollNext( + new Collector<SeaTunnelRow>() { + @Override + public void collect(SeaTunnelRow row) { + records.add(row); + } + + @Override + public Object getCheckpointLock() { + return records; + } + }); + verify(readerContext).signalNoMoreElement(); + } + assertEquals(1, records.size()); + assertEquals(1, records.get(0).getField(0)); + assertEquals("rest-row", records.get(0).getField(1)); + assertEquals("test_db.test_table", records.get(0).getTableId()); + } + + @Test + void testDlfSignedRequestAgainstLocalServer() throws Exception { + RESTCatalogServer server = + new RESTCatalogServer( + directory.toString(), + DLFAuthProvider.buildAKToken( + "test-id", "test-secret", "test-session", "cn-hangzhou"), + serverConfig(), + "managed-warehouse"); + server.start(); + ClassLoader previous = Thread.currentThread().getContextClassLoader(); + try { + Map<String, Object> options = options(server.getUrl()); + options.put("token.provider", "dlf"); + options.put("dlf.access-key-id", "test-id"); + options.put("dlf.access-key-secret", "test-secret"); + options.put("dlf.security-token", "test-session"); + options.put("dlf.region", "cn-hangzhou"); + try (Catalog catalog = + new PaimonCatalogLoader(new PaimonConfig(ReadonlyConfig.fromMap(options))) + .loadCatalog()) { + catalog.createDatabase("dlf_test", false); + assertTrue(catalog.listDatabases().contains("dlf_test")); + } + } finally { + Thread.currentThread().setContextClassLoader(previous); + server.shutdown(); + } + } + + private ConfigResponse serverConfig() { + return new ConfigResponse( + Collections.singletonMap(RESTCatalogInternalOptions.PREFIX.key(), "test"), + Collections.emptyMap()); + } + + private Map<String, Object> options(String uri) { + Map<String, Object> options = new HashMap<>(); + options.put("catalog_type", "rest"); + options.put("catalog_uri", uri); + options.put("warehouse", "managed-warehouse"); + options.put("database", "test_db"); + options.put("table", "test_table"); + options.put("paimon.table.write-props", Collections.singletonMap("bucket", "1")); + return options; + } + + private CatalogTable catalogTable() { + TableSchema schema = + TableSchema.builder() + .column( + PhysicalColumn.of( + "id", BasicType.INT_TYPE, (Long) null, false, null, null)) + .column( + PhysicalColumn.of( + "value", + BasicType.STRING_TYPE, + (Long) null, + true, + null, + null)) + .primaryKey(PrimaryKey.of("pk", Collections.singletonList("id"))) + .build(); + return CatalogTable.of( + TableIdentifier.of("paimon", "test_db", null, "test_table"), + schema, + Collections.emptyMap(), + Collections.emptyList(), + "REST test"); + } +}
