This is an automated email from the ASF dual-hosted git repository.
czy006 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/amoro.git
The following commit(s) were added to refs/heads/master by this push:
new 756fbec0b [AMORO-3591] Support Apache Paimon With OSS Object Store
(#3593)
756fbec0b is described below
commit 756fbec0be93e4c47dc7b9ef2b44fe18c49e9281
Author: ConradJam <[email protected]>
AuthorDate: Thu Jun 12 19:49:00 2025 +0800
[AMORO-3591] Support Apache Paimon With OSS Object Store (#3593)
* [AMORO-3591] Support Apache Paimon With OSS Object Store
* fix pom
---
.../dashboard/controller/CatalogController.java | 26 ++++++++++++++++------
amoro-format-paimon/pom.xml | 13 ++++++++++-
.../amoro/formats/paimon/PaimonCatalogFactory.java | 26 ++++++++++++++++------
pom.xml | 13 +++++++++++
4 files changed, 63 insertions(+), 15 deletions(-)
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/CatalogController.java
b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/CatalogController.java
index fbbd2b651..7c3e044ab 100644
---
a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/CatalogController.java
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/CatalogController.java
@@ -394,12 +394,16 @@ public class CatalogController {
STORAGE_CONFIGS_KEY_S3_ENDPOINT);
} else if (STORAGE_CONFIGS_VALUE_TYPE_OSS.equals(storageType)) {
CatalogUtil.copyProperty(
+ catalogMeta.getStorageConfigs(),
catalogMeta.getCatalogProperties(),
- storageConfig,
AliyunProperties.OSS_ENDPOINT,
STORAGE_CONFIGS_KEY_OSS_ENDPOINT);
+ CatalogUtil.copyProperty(
+ catalogMeta.getStorageConfigs(),
+ catalogMeta.getCatalogProperties(),
+ "fs.oss.endpoint",
+ STORAGE_CONFIGS_KEY_OSS_ENDPOINT);
}
-
return storageConfig;
}
@@ -475,11 +479,19 @@ public class CatalogController {
STORAGE_CONFIGS_KEY_S3_ENDPOINT,
S3FileIOProperties.ENDPOINT);
} else if (storageType.equals(STORAGE_CONFIGS_VALUE_TYPE_OSS)) {
- CatalogUtil.copyProperty(
- info.getStorageConfig(),
- catalogMeta.getCatalogProperties(),
- STORAGE_CONFIGS_KEY_OSS_ENDPOINT,
- AliyunProperties.OSS_ENDPOINT);
+ if (TableFormat.valueOf(tableFormats) == TableFormat.ICEBERG) {
+ CatalogUtil.copyProperty(
+ info.getStorageConfig(),
+ catalogMeta.getCatalogProperties(),
+ STORAGE_CONFIGS_KEY_OSS_ENDPOINT,
+ AliyunProperties.OSS_ENDPOINT);
+ } else if (TableFormat.valueOf(tableFormats) == PAIMON) {
+ CatalogUtil.copyProperty(
+ info.getStorageConfig(),
+ catalogMeta.getCatalogProperties(),
+ STORAGE_CONFIGS_KEY_OSS_ENDPOINT,
+ "fs.oss.endpoint");
+ }
} else {
throw new RuntimeException("Invalid storage type " + storageType);
}
diff --git a/amoro-format-paimon/pom.xml b/amoro-format-paimon/pom.xml
index 49e6dcf66..e2388d623 100644
--- a/amoro-format-paimon/pom.xml
+++ b/amoro-format-paimon/pom.xml
@@ -54,10 +54,21 @@
<artifactId>paimon-s3</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-oss</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-aliyun</artifactId>
+ <scope>${aliyun-sdk-dependency-scope}</scope>
+ </dependency>
+
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.12</artifactId>
- <version>3.3.2</version>
+ <version>3.3.4</version>
<scope>provided</scope>
</dependency>
</dependencies>
diff --git
a/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonCatalogFactory.java
b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonCatalogFactory.java
index 4787628ac..cb6dee58f 100644
---
a/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonCatalogFactory.java
+++
b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonCatalogFactory.java
@@ -31,6 +31,8 @@ import org.apache.paimon.catalog.FileSystemCatalogFactory;
import org.apache.paimon.hive.HiveCatalogOptions;
import org.apache.paimon.options.CatalogOptions;
import org.apache.paimon.options.Options;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.File;
import java.net.URL;
@@ -39,8 +41,13 @@ import java.util.Optional;
public class PaimonCatalogFactory implements FormatCatalogFactory {
+ private static final Logger LOG =
LoggerFactory.getLogger(PaimonCatalogFactory.class);
+
public static final String PAIMON_S3_ACCESS_KEY = "s3.access-key";
public static final String PAIMON_S3_SECRET_KEY = "s3.secret-key";
+ public static final String PAIMON_OSS_ACCESS_KEY = "fs.oss.accessKeyId";
+ public static final String PAIMON_OSS_SECRET_KEY = "fs.oss.accessKeySecret";
+ public static final String PAIMON_OSS_ENDPOINT = "fs.oss.endpoint";
@Override
public PaimonCatalog create(
@@ -50,19 +57,24 @@ public class PaimonCatalogFactory implements
FormatCatalogFactory {
// if format table enabled, paimon will load hive orc/parquet/csv table to
paimon table
catalogProperties.put(CatalogOptions.FORMAT_TABLE_ENABLED.key(), "false");
catalogProperties.putAll(properties);
-
hiveSiteLocation.ifPresent(
url ->
catalogProperties.put(
HiveCatalogOptions.HIVE_CONF_DIR.key(), new
File(url.getPath()).getParent()));
-
if (CatalogMetaProperties.AUTH_CONFIGS_VALUE_TYPE_AK_SK.equalsIgnoreCase(
metaStore.getAuthMethod())) {
- // s3.access-key, s3.secret-key
- catalogProperties.put(PAIMON_S3_ACCESS_KEY, metaStore.getAccessKey());
- catalogProperties.put(PAIMON_S3_SECRET_KEY, metaStore.getSecretKey());
- Catalog catalog = paimonCatalog(catalogProperties, new Configuration());
- return new PaimonCatalog(catalog, name);
+ if
(CatalogMetaProperties.STORAGE_CONFIGS_VALUE_TYPE_S3.equals(metastoreType)) {
+ // s3.access-key, s3.secret-key
+ catalogProperties.put(PAIMON_S3_ACCESS_KEY, metaStore.getAccessKey());
+ catalogProperties.put(PAIMON_S3_SECRET_KEY, metaStore.getSecretKey());
+ Catalog catalog = paimonCatalog(catalogProperties, new
Configuration());
+ return new PaimonCatalog(catalog, name);
+ } else {
+ catalogProperties.put(PAIMON_OSS_ACCESS_KEY, metaStore.getAccessKey());
+ catalogProperties.put(PAIMON_OSS_SECRET_KEY, metaStore.getSecretKey());
+ Catalog catalog = paimonCatalog(catalogProperties, new
Configuration());
+ return new PaimonCatalog(catalog, name);
+ }
} else {
Catalog catalog = paimonCatalog(catalogProperties,
metaStore.getConfiguration());
return new PaimonCatalog(catalog, name);
diff --git a/pom.xml b/pom.xml
index f8e83f1f9..bd4b415b6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -381,6 +381,19 @@
<version>${parquet-jackson.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-aliyun</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>${aliyun-sdk-dependency-scope}</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-oss</artifactId>
+ <version>${paimon.version}</version>
+ </dependency>
+
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-auth</artifactId>
