This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/doris.git
commit bda2042c7824073596f5f22bb80189b45b51cddf Author: wuwenchi <[email protected]> AuthorDate: Mon Feb 19 10:09:41 2024 +0800 [enhancement](paimon)add obs filesystem for paimon (#31061) --- docs/en/docs/lakehouse/multi-catalog/paimon.md | 20 ++++++++++++++++++++ docs/zh-CN/docs/lakehouse/multi-catalog/paimon.md | 20 ++++++++++++++++++++ .../datasource/paimon/PaimonFileExternalCatalog.java | 10 ++++++++++ 3 files changed, 50 insertions(+) diff --git a/docs/en/docs/lakehouse/multi-catalog/paimon.md b/docs/en/docs/lakehouse/multi-catalog/paimon.md index c4c7bb64860..49012cf7318 100644 --- a/docs/en/docs/lakehouse/multi-catalog/paimon.md +++ b/docs/en/docs/lakehouse/multi-catalog/paimon.md @@ -93,6 +93,26 @@ CREATE CATALOG `paimon_s3` PROPERTIES ( ); ``` +#### OBS + +> Note that. +> +> user need download [paimon-s3-0.6.0-incubating.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-s3/0.6.0-incubating/paimon-s3-0.6.0-incubating.jar) +> +> Place it in directory `${DORIS_HOME}/be/lib/java_extensions/preload-extensions` and restart be +> +> Starting from version 2.0.2, this file can be placed in BE's `custom_lib/` directory (if it does not exist, just create it manually) to prevent the file from being lost due to the replacement of the lib directory when upgrading the cluster. + +```sql +CREATE CATALOG `paimon_obs` PROPERTIES ( + "type" = "paimon", + "warehouse" = "obs://bucket_name/paimon", + "obs.endpoint"="obs.cn-north-4.myhuaweicloud.com", + "obs.access_key"="ak", + "obs.secret_key"="sk" +); +``` + #### COS > Note that. diff --git a/docs/zh-CN/docs/lakehouse/multi-catalog/paimon.md b/docs/zh-CN/docs/lakehouse/multi-catalog/paimon.md index 79632be6772..d196df7a87e 100644 --- a/docs/zh-CN/docs/lakehouse/multi-catalog/paimon.md +++ b/docs/zh-CN/docs/lakehouse/multi-catalog/paimon.md @@ -93,6 +93,26 @@ CREATE CATALOG `paimon_s3` PROPERTIES ( ); ``` +#### OBS + +> 注意： +> +> 用户需要手动下载[paimon-s3-0.6.0-incubating.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-s3/0.6.0-incubating/paimon-s3-0.6.0-incubating.jar) + +> 放在 `${DORIS_HOME}/be/lib/java_extensions/preload-extensions` 目录下并重启be。 +> +> 从 2.0.2 版本起，可以将这个文件放置在BE的 `custom_lib/` 目录下（如不存在，手动创建即可），以防止升级集群时因为 lib 目录被替换而导致文件丢失。 + +```sql +CREATE CATALOG `paimon_obs` PROPERTIES ( + "type" = "paimon", + "warehouse" = "obs://bucket_name/paimon", + "obs.endpoint"="obs.cn-north-4.myhuaweicloud.com", + "obs.access_key"="ak", + "obs.secret_key"="sk" +); +``` + #### COS > 注意： diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonFileExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonFileExternalCatalog.java index 9e8fa2dc455..62b29c98f3a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonFileExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonFileExternalCatalog.java @@ -20,6 +20,7 @@ package org.apache.doris.datasource.paimon; import org.apache.doris.datasource.CatalogProperty; import org.apache.doris.datasource.property.PropertyConverter; import org.apache.doris.datasource.property.constants.CosProperties; +import org.apache.doris.datasource.property.constants.ObsProperties; import org.apache.doris.datasource.property.constants.PaimonProperties; import org.apache.logging.log4j.LogManager; @@ -68,6 +69,15 @@ public class PaimonFileExternalCatalog extends PaimonExternalCatalog { properties.get(CosProperties.SECRET_KEY)); options.put(PaimonProperties.WAREHOUSE, options.get(PaimonProperties.WAREHOUSE).replace("cosn://", "s3://")); + } else if (properties.containsKey(ObsProperties.ENDPOINT)) { + options.put(PaimonProperties.PAIMON_S3_ENDPOINT, + properties.get(ObsProperties.ENDPOINT)); + options.put(PaimonProperties.PAIMON_S3_ACCESS_KEY, + properties.get(ObsProperties.ACCESS_KEY)); + options.put(PaimonProperties.PAIMON_S3_SECRET_KEY, + properties.get(ObsProperties.SECRET_KEY)); + options.put(PaimonProperties.WAREHOUSE, + options.get(PaimonProperties.WAREHOUSE).replace("obs://", "s3://")); } if (properties.containsKey(PropertyConverter.USE_PATH_STYLE)) { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
