This is an automated email from the ASF dual-hosted git repository.
kirs pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 1cdab3de053 branch-3.1: [chore](paimon)Unify object storage access via
HDFS protocol #54776 (#54979)
1cdab3de053 is described below
commit 1cdab3de053846e95b9216eee08f41c7742f90db
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Aug 19 17:50:45 2025 +0800
branch-3.1: [chore](paimon)Unify object storage access via HDFS protocol
#54776 (#54979)
Cherry-picked from #54776
Co-authored-by: Calvin Kirs <[email protected]>
---
.../docker-compose/hive/hadoop-hive-3x.env.tpl | 2 +
.../docker-compose/kerberos/hadoop-hive.env.tpl | 2 +
.../metastore/AbstractPaimonProperties.java | 31 -------
.../metastore/HMSGlueMetaStoreProperties.java | 4 +
.../storage/AbstractS3CompatibleProperties.java | 1 +
.../datasource/property/storage/COSProperties.java | 4 +-
.../datasource/property/storage/OSSProperties.java | 43 ++++++----
.../property/metastore/PaimonCatalogTest.java | 95 ++++++++++++++++++++++
8 files changed, 135 insertions(+), 47 deletions(-)
diff --git a/docker/thirdparties/docker-compose/hive/hadoop-hive-3x.env.tpl
b/docker/thirdparties/docker-compose/hive/hadoop-hive-3x.env.tpl
index c8afe5046d0..4675fbf8053 100644
--- a/docker/thirdparties/docker-compose/hive/hadoop-hive-3x.env.tpl
+++ b/docker/thirdparties/docker-compose/hive/hadoop-hive-3x.env.tpl
@@ -20,6 +20,7 @@
HIVE_SITE_CONF_hive_metastore_event_db_notification_api_auth=false
HIVE_SITE_CONF_hive_metastore_dml_events=true
HIVE_SITE_CONF_hive_metastore_transactional_event_listeners=org.apache.hive.hcatalog.listener.DbNotificationListener
HIVE_SITE_CONF_hive_stats_column_autogather=false
+HIVE_SITE_CONF_fs_s3_impl=org.apache.hadoop.fs.s3a.S3AFileSystem
HIVE_SITE_CONF_fs_s3a_impl=org.apache.hadoop.fs.s3a.S3AFileSystem
HIVE_SITE_CONF_fs_s3a_access_key=${AWSAk}
HIVE_SITE_CONF_fs_s3a_secret_key=${AWSSk}
@@ -33,6 +34,7 @@ HIVE_SITE_CONF_fs_cosn_userinfo_secretId=${COSAk}
HIVE_SITE_CONF_fs_cosn_userinfo_secretKey=${COSSk}
HIVE_SITE_CONF_fs_cosn_bucket_region=${COSRegion}
HIVE_SITE_CONF_fs_cosn_impl=org.apache.hadoop.fs.CosFileSystem
+HIVE_SITE_CONF_fs_cos_impl=org.apache.hadoop.fs.CosFileSystem
HIVE_SITE_CONF_fs_AbstractFileSystem_cosn_impl=org.apache.hadoop.fs.CosN
HIVE_SITE_CONF_fs_oss_impl=org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem
HIVE_SITE_CONF_fs_oss_accessKeyId=${OSSAk}
diff --git a/docker/thirdparties/docker-compose/kerberos/hadoop-hive.env.tpl
b/docker/thirdparties/docker-compose/kerberos/hadoop-hive.env.tpl
index f06c1ef7d7a..3c453ff0fbd 100644
--- a/docker/thirdparties/docker-compose/kerberos/hadoop-hive.env.tpl
+++ b/docker/thirdparties/docker-compose/kerberos/hadoop-hive.env.tpl
@@ -64,6 +64,7 @@
YARN_CONF_yarn_resourcemanager_webapp_address=${HOST}:${YARN_RM_WEBAPP_PORT}
YARN_CONF_yarn_nodemanager_localizer_address=${HOST}:${YARN_NM_LOCAL_PORT}
YARN_CONF_yarn_nodemanager_webapp_address=${HOST}:${YARN_NM_WEBAPP_PORT}
+HIVE_SITE_CONF_fs_s3_impl=org.apache.hadoop.fs.s3a.S3AFileSystem
HIVE_SITE_CONF_fs_s3a_impl=org.apache.hadoop.fs.s3a.S3AFileSystem
HIVE_SITE_CONF_fs_s3a_access_key=${AWSAk}
HIVE_SITE_CONF_fs_s3a_secret_key=${AWSSk}
@@ -77,6 +78,7 @@ HIVE_SITE_CONF_fs_cosn_userinfo_secretId=${COSAk}
HIVE_SITE_CONF_fs_cosn_userinfo_secretKey=${COSSk}
HIVE_SITE_CONF_fs_cosn_bucket_region=${COSRegion}
HIVE_SITE_CONF_fs_cosn_impl=org.apache.hadoop.fs.CosFileSystem
+HIVE_SITE_CONF_fs_cos_impl=org.apache.hadoop.fs.CosFileSystem
HIVE_SITE_CONF_fs_AbstractFileSystem_cosn_impl=org.apache.hadoop.fs.CosN
HIVE_SITE_CONF_fs_oss_impl=org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem
HIVE_SITE_CONF_fs_oss_accessKeyId=${OSSAk}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java
index 7602758d2b0..040d04a1aad 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java
@@ -19,7 +19,6 @@ package org.apache.doris.datasource.property.metastore;
import org.apache.doris.common.security.authentication.ExecutionAuthenticator;
import org.apache.doris.datasource.property.ConnectorProperty;
-import org.apache.doris.datasource.property.storage.S3Properties;
import org.apache.doris.datasource.property.storage.StorageProperties;
import lombok.Getter;
@@ -59,34 +58,6 @@ public abstract class AbstractPaimonProperties extends
MetastoreProperties {
public abstract Catalog initializeCatalog(String catalogName,
List<StorageProperties> storagePropertiesList);
- /**
- * Adapt S3 storage properties for Apache Paimon's S3 file system.
- *
- * <p>Paimon's S3 file system does not follow the standard
Hadoop-compatible
- * configuration keys (like fs.s3a.access.key). Instead, it expects
specific
- * keys such as "s3.access.key", "s3.secret.key", etc.
- *
- * <p>Therefore, we explicitly map our internal S3 configuration (usually
designed
- * for HDFS-compatible systems) to Paimon's expected format.
- *
- * <p>See: org.apache.paimon.s3.S3Loader
- *
- * @param storagePropertiesList the list of configured storage backends
- */
- protected void appendS3PropertiesIsNeeded(List<StorageProperties>
storagePropertiesList) {
-
- S3Properties s3Properties = (S3Properties)
storagePropertiesList.stream()
- .filter(storageProperties -> storageProperties.getType() ==
StorageProperties.Type.S3)
- .findFirst()
- .orElse(null);
- if (s3Properties != null) {
- catalogOptions.set("s3.access.key", s3Properties.getSecretKey());
- catalogOptions.set("s3.secret.key", s3Properties.getAccessKey());
- catalogOptions.set("s3.endpoint", s3Properties.getEndpoint());
- catalogOptions.set("s3.region", s3Properties.getRegion());
- }
- }
-
protected void appendCatalogOptions(List<StorageProperties>
storagePropertiesList) {
if (StringUtils.isNotBlank(warehouse)) {
catalogOptions.set(CatalogOptions.WAREHOUSE.key(), warehouse);
@@ -102,8 +73,6 @@ public abstract class AbstractPaimonProperties extends
MetastoreProperties {
}
}
});
-
- appendS3PropertiesIsNeeded(storagePropertiesList);
}
/**
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSGlueMetaStoreProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSGlueMetaStoreProperties.java
index 477dba85ebf..1f31fe44dc2 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSGlueMetaStoreProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSGlueMetaStoreProperties.java
@@ -41,6 +41,8 @@ public class HMSGlueMetaStoreProperties extends
AbstractHMSProperties {
public static final int DEFAULT_MAX_RETRY = 5;
public static final String AWS_GLUE_SOCKET_TIMEOUT_KEY =
"aws.glue.socket-timeout";
public static final int DEFAULT_SOCKET_TIMEOUT =
ClientConfiguration.DEFAULT_SOCKET_TIMEOUT;
+ public static final String
AWS_CATALOG_CREDENTIALS_PROVIDER_FACTORY_CLASS_KEY =
+ "aws.catalog.credentials.provider.factory.class";
// ========== Fields ==========
private AWSGlueMetaStoreBaseProperties baseProperties;
@@ -104,6 +106,8 @@ public class HMSGlueMetaStoreProperties extends
AbstractHMSProperties {
hiveConf.set(AWS_GLUE_CONNECTION_TIMEOUT_KEY,
String.valueOf(awsGlueConnectionTimeout));
hiveConf.set(AWS_GLUE_SOCKET_TIMEOUT_KEY,
String.valueOf(awsGlueSocketTimeout));
hiveConf.set(AWS_GLUE_CATALOG_SEPARATOR_KEY, awsGlueCatalogSeparator);
+ hiveConf.set(AWS_CATALOG_CREDENTIALS_PROVIDER_FACTORY_CLASS_KEY,
+
"com.amazonaws.glue.catalog.credentials.ConfigurationAWSCredentialsProviderFactory");
hiveConf.set("hive.metastore.type", "glue");
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java
index 000bc75c529..509ebd81e87 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java
@@ -303,6 +303,7 @@ public abstract class AbstractS3CompatibleProperties
extends StorageProperties i
hadoopStorageConfig.set("fs.s3.impl",
"org.apache.hadoop.fs.s3a.S3AFileSystem");
hadoopStorageConfig.set("fs.s3a.impl",
"org.apache.hadoop.fs.s3a.S3AFileSystem");
hadoopStorageConfig.set("fs.s3a.endpoint", getEndpoint());
+ hadoopStorageConfig.set("fs.s3a.endpoint.region", getRegion());
hadoopStorageConfig.set("fs.s3a.access.key", getAccessKey());
hadoopStorageConfig.set("fs.s3a.secret.key", getSecretKey());
hadoopStorageConfig.set("fs.s3a.connection.maximum",
getMaxConnections());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java
index 0e0f37e5566..f1820abb2b3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java
@@ -130,8 +130,8 @@ public class COSProperties extends
AbstractS3CompatibleProperties {
@Override
public void initializeHadoopStorageConfig() {
super.initializeHadoopStorageConfig();
- hadoopStorageConfig.set("fs.cos.impl",
"org.apache.hadoop.fs.CosFileSystem");
- hadoopStorageConfig.set("fs.cosn.impl",
"org.apache.hadoop.fs.CosFileSystem");
+ hadoopStorageConfig.set("fs.cos.impl",
"org.apache.hadoop.fs.s3a.S3AFileSystem");
+ hadoopStorageConfig.set("fs.cosn.impl",
"org.apache.hadoop.fs.s3a.S3AFileSystem");
hadoopStorageConfig.set("fs.cosn.bucket.region", region);
hadoopStorageConfig.set("fs.cosn.userinfo.secretId", accessKey);
hadoopStorageConfig.set("fs.cosn.userinfo.secretKey", secretKey);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
index 45208e6f3d3..07d56648498 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
@@ -97,6 +97,9 @@ public class OSSProperties extends
AbstractS3CompatibleProperties {
private static final List<String> URI_KEYWORDS = Arrays.asList("uri",
"warehouse");
+ private static List<String> DLF_TYPE_KEYWORDS =
Arrays.asList("hive.metastore.type",
+ "iceberg.catalog.type", "paimon.catalog.type");
+
protected OSSProperties(Map<String, String> origProps) {
super(Type.OSS, origProps);
}
@@ -128,7 +131,9 @@ public class OSSProperties extends
AbstractS3CompatibleProperties {
if (StringUtils.isNotBlank(value)) {
return true;
}
-
+ if (isDlfMSType(origProps)) {
+ return true;
+ }
Optional<String> uriValue = origProps.entrySet().stream()
.filter(e -> URI_KEYWORDS.stream()
.anyMatch(key -> key.equalsIgnoreCase(e.getKey())))
@@ -155,22 +160,32 @@ public class OSSProperties extends
AbstractS3CompatibleProperties {
return isAliyunOss || isAmazonS3 || isDls;
}
+ private static boolean isDlfMSType(Map<String, String> params) {
+ return DLF_TYPE_KEYWORDS.stream()
+ .anyMatch(key -> params.containsKey(key) &&
StringUtils.isNotBlank(params.get(key))
+ && StringUtils.equalsIgnoreCase("dlf",
params.get(key)));
+ }
+
@Override
protected void setEndpointIfPossible() {
if (StringUtils.isBlank(this.endpoint) &&
StringUtils.isNotBlank(this.region)) {
- Optional<String> uriValueOpt = origProps.entrySet().stream()
- .filter(e -> URI_KEYWORDS.stream()
- .anyMatch(key -> key.equalsIgnoreCase(e.getKey())))
- .map(Map.Entry::getValue)
- .filter(Objects::nonNull)
- .filter(OSSProperties::isKnownObjectStorage)
- .findFirst();
- if (uriValueOpt.isPresent()) {
- String uri = uriValueOpt.get();
- // If the URI does not start with http(s), derive endpoint
from region
- // (http(s) URIs are handled by separate logic elsewhere)
- if (!uri.startsWith("http://";) && !uri.startsWith("https://";))
{
- this.endpoint = getOssEndpoint(region,
BooleanUtils.toBoolean(dlfAccessPublic));
+ if (isDlfMSType(origProps)) {
+ this.endpoint = getOssEndpoint(region,
BooleanUtils.toBoolean(dlfAccessPublic));
+ } else {
+ Optional<String> uriValueOpt = origProps.entrySet().stream()
+ .filter(e -> URI_KEYWORDS.stream()
+ .anyMatch(key ->
key.equalsIgnoreCase(e.getKey())))
+ .map(Map.Entry::getValue)
+ .filter(Objects::nonNull)
+ .filter(OSSProperties::isKnownObjectStorage)
+ .findFirst();
+ if (uriValueOpt.isPresent()) {
+ String uri = uriValueOpt.get();
+ // If the URI does not start with http(s), derive endpoint
from region
+ // (http(s) URIs are handled by separate logic elsewhere)
+ if (!uri.startsWith("http://";) &&
!uri.startsWith("https://";)) {
+ this.endpoint = getOssEndpoint(region,
BooleanUtils.toBoolean(dlfAccessPublic));
+ }
}
}
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonCatalogTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonCatalogTest.java
new file mode 100644
index 00000000000..59766ef9902
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonCatalogTest.java
@@ -0,0 +1,95 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.metastore;
+
+import org.apache.doris.datasource.property.storage.StorageProperties;
+
+import org.apache.paimon.catalog.Catalog;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@Disabled("only used for your local test")
+public class PaimonCatalogTest {
+ @Test
+ public void testNameSpace() throws Exception {
+ Map<String, String> pa = new HashMap<>();
+ pa.put("type", "paimon");
+ pa.put("paimon.catalog.type", "hms");
+ pa.put("hive.metastore.uris", "thrift://172.20.48.119:9383");
+ pa.put("warehouse", "s3a://doris/paimon_warehouse");
+ pa.put("s3.region", "ap-east-1");
+
+ // User must provide real Access Key / Secret Key to enable
initialization
+ pa.put("s3.access_key", "");
+ pa.put("s3.secret_key", "");
+ pa.put("s3.endpoint", "s3.ap-east-1.amazonaws.com");
+
+ Catalog catalog = initCatalog(pa);
+ if (catalog != null) {
+ catalog.listDatabases().forEach(System.out::println);
+ }
+ }
+
+ /**
+ * Initializes a Paimon HMS Catalog.
+ * <p>
+ * Initialization is skipped by default. Users must provide valid S3
+ * Access Key and Secret Key in the configuration map to enable it.
+ * <p>
+ * Steps:
+ * 1. Validate that credentials are provided.
+ * 2. Normalize and check metastore properties.
+ * 3. Create storage properties.
+ * 4. Initialize and return the Catalog instance.
+ *
+ * @param params A map containing the configuration parameters.
+ * @return Catalog instance if initialized, or {@code null} if skipped.
+ * @throws Exception If initialization fails.
+ */
+ private Catalog initCatalog(Map<String, String> params) throws Exception {
+ if (isDisabled(params)) {
+ System.out.println("Catalog initialization skipped: Missing valid
S3 Access Key/Secret Key.");
+ return null;
+ }
+
+ AbstractPaimonProperties metaStoreProps =
+ (AbstractPaimonProperties) MetastoreProperties.create(params);
+
+ metaStoreProps.initNormalizeAndCheckProps();
+
+ List<StorageProperties> storageProps =
StorageProperties.createAll(params);
+
+ return metaStoreProps.initializeCatalog("paimon_catalog",
storageProps);
+ }
+
+ /**
+ * Checks if initialization should be skipped due to missing credentials.
+ *
+ * @param params The configuration parameters.
+ * @return {@code true} if missing AK/SK, {@code false} otherwise.
+ */
+ private boolean isDisabled(Map<String, String> params) {
+ String ak = params.get("s3.access_key");
+ String sk = params.get("s3.secret_key");
+ return ak == null || ak.isEmpty() || sk == null || sk.isEmpty();
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
