This is an automated email from the ASF dual-hosted git repository.
kirs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 28d6b3950e5 [paimon](obs) support pfs use in paimon catalog (#60995)
28d6b3950e5 is described below
commit 28d6b3950e51715de0fa12ac638c27d8a6dc52ac
Author: zhangstar333 <[email protected]>
AuthorDate: Wed Mar 11 15:31:16 2026 +0800
[paimon](obs) support pfs use in paimon catalog (#60995)
### What problem does this PR solve?
Problem Summary:
```
mysql> create catalog if not exists paimon_self2 properties (
-> "type" = "paimon",
-> "paimon.catalog.type"="filesystem",
-> "warehouse" = "obs://paimon-test/test/obs_hive/",
-> "obs.access_key"="aaaaa",
-> "obs.secret_key"="ssssss",
-> "obs.endpoint"="obs.cn-east-3.myhuaweicloud.com"
-> );
Query OK, 0 rows affected (0.06 sec)
mysql> switch paimon_self2;
Query OK, 0 rows affected (0.00 sec)
mysql> show databases;
+--------------------+
| Database |
+--------------------+
| db1 |
| information_schema |
| mysql |
| obs_test |
+--------------------+
4 rows in set (1.41 sec)
mysql> use db1;
Reading table information for completion of table and column names
You can turn off this feature to get a quicker startup with -A
Database changed
mysql> show tables;
+------------------+
| Tables_in_db1 |
+------------------+
| binary_size_test |
+------------------+
1 row in set (0.00 sec)
mysql> select * from binary_size_test;
+---------+---------------+-------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
| test_id | size_category | target_size | binary_data
```
### Release note
---
fe/be-java-extensions/hadoop-deps/pom.xml | 12 ++++++++++++
.../datasource/property/storage/OBSProperties.java | 20 +++++++++++++++++++-
2 files changed, 31 insertions(+), 1 deletion(-)
diff --git a/fe/be-java-extensions/hadoop-deps/pom.xml
b/fe/be-java-extensions/hadoop-deps/pom.xml
index 102a92d356e..54591563091 100644
--- a/fe/be-java-extensions/hadoop-deps/pom.xml
+++ b/fe/be-java-extensions/hadoop-deps/pom.xml
@@ -71,6 +71,10 @@ under the License.
<groupId>org.apache.hadoop.thirdparty</groupId>
<artifactId>hadoop-shaded-protobuf_3_25</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.huaweicloud</groupId>
+ <artifactId>hadoop-huaweicloud</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
@@ -95,6 +99,14 @@ under the License.
</dependency>
</dependencies>
+ <repositories>
+ <!-- for huawei obs sdk -->
+ <repository>
+ <id>huawei-obs-sdk</id>
+
<url>https://repo.huaweicloud.com/repository/maven/huaweicloudsdk/</url>
+ </repository>
+ </repositories>
+
<build>
<plugins>
<plugin>
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java
index 6b3cafe99fd..0f4f2d6d3f6 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java
@@ -169,10 +169,28 @@ public class OBSProperties extends
AbstractS3CompatibleProperties {
return null;
}
+ private static final boolean OBS_FILE_SYSTEM_AVAILABLE =
+ isClassAvailable("org.apache.hadoop.fs.obs.OBSFileSystem");
+
+ private static boolean isClassAvailable(String className) {
+ try {
+ Class.forName(className, false,
OBSProperties.class.getClassLoader());
+ return true;
+ } catch (ClassNotFoundException e) {
+ return false;
+ }
+ }
+
@Override
public void initializeHadoopStorageConfig() {
super.initializeHadoopStorageConfig();
- hadoopStorageConfig.set("fs.obs.impl",
"org.apache.hadoop.fs.s3a.S3AFileSystem");
+ // obs is not compatible with s3a well; prefer native OBSFileSystem if
available on the classpath
+ if (OBS_FILE_SYSTEM_AVAILABLE) {
+ hadoopStorageConfig.set("fs.obs.impl",
"org.apache.hadoop.fs.obs.OBSFileSystem");
+ hadoopStorageConfig.set("fs.AbstractFileSystem.obs.impl",
"org.apache.hadoop.fs.obs.OBS");
+ } else {
+ hadoopStorageConfig.set("fs.obs.impl",
"org.apache.hadoop.fs.s3a.S3AFileSystem");
+ }
hadoopStorageConfig.set("fs.obs.access.key", accessKey);
hadoopStorageConfig.set("fs.obs.secret.key", secretKey);
hadoopStorageConfig.set("fs.obs.endpoint", endpoint);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
