This is an automated email from the ASF dual-hosted git repository.
CalvinKirs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new ceeb9bc09a9 [feat](Iceberg)Rest & S3Table Support Iam-role (#60498)
ceeb9bc09a9 is described below
commit ceeb9bc09a90ef5d0c6bf4851a7cc51090b4d1d7
Author: Calvin Kirs <[email protected]>
AuthorDate: Thu May 7 11:02:49 2026 +0800
[feat](Iceberg)Rest & S3Table Support Iam-role (#60498)
FYI https://github.com/apache/doris/pull/59893
The initial implementation was contributed by https://github.com/Sbaia .
Many thanks for his contributions.
---
.../s3tables/CustomAwsCredentialsProvider.java | 43 ---
.../common/AwsCredentialsProviderFactory.java | 28 ++
.../common/IcebergAwsAssumeRoleProperties.java | 52 ++++
.../IcebergAwsClientCredentialsProperties.java | 144 ++++++++++
.../metastore/AbstractIcebergProperties.java | 5 +
.../property/metastore/IcebergRestProperties.java | 72 ++++-
.../IcebergS3TablesMetaStoreProperties.java | 33 ++-
.../datasource/property/storage/S3Properties.java | 29 +-
.../metastore/IcebergRestPropertiesTest.java | 303 +++++++++++++++++++++
.../IcebergS3TablesMetaStorePropertiesTest.java | 272 ++++++++++++++++++
.../property/storage/S3PropertiesTest.java | 13 +
.../doris/datasource/s3tables/S3TablesTest.java | 9 -
...rg_s3tables_catalog_credentials_provider.groovy | 105 +++++++
13 files changed, 1028 insertions(+), 80 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/s3tables/CustomAwsCredentialsProvider.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/s3tables/CustomAwsCredentialsProvider.java
deleted file mode 100644
index f76c8c3cc9b..00000000000
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/s3tables/CustomAwsCredentialsProvider.java
+++ /dev/null
@@ -1,43 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.datasource.iceberg.s3tables;
-
-import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
-import software.amazon.awssdk.auth.credentials.AwsCredentials;
-import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
-
-import java.util.Map;
-
-public class CustomAwsCredentialsProvider implements AwsCredentialsProvider {
- private final String accessKeyId;
- private final String secretAccessKey;
-
- public CustomAwsCredentialsProvider(String accessKeyId, String
secretAccessKey) {
- this.accessKeyId = accessKeyId;
- this.secretAccessKey = secretAccessKey;
- }
-
- @Override
- public AwsCredentials resolveCredentials() {
- return AwsBasicCredentials.create(accessKeyId, secretAccessKey);
- }
-
- public static CustomAwsCredentialsProvider create(Map<String, String>
props) {
- return new CustomAwsCredentialsProvider(props.get("s3.access-key-id"),
props.get("s3.secret-access-key"));
- }
-}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/AwsCredentialsProviderFactory.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/AwsCredentialsProviderFactory.java
index d2fd4a80d9e..8669488cf1a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/AwsCredentialsProviderFactory.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/AwsCredentialsProviderFactory.java
@@ -177,4 +177,32 @@ public final class AwsCredentialsProviderFactory {
"AWS SDK V2 does not support credentials provider
mode: " + mode);
}
}
+
+ /**
+ * Get the AWS credentials provider class name.
+ * For DEFAULT mode, returns AWS SDK native DefaultCredentialsProvider.
+ * For other modes, returns the specific provider class name.
+ */
+ public static String getV2ClassName(AwsCredentialsProviderMode mode) {
+ switch (mode) {
+ case ENV:
+ return EnvironmentVariableCredentialsProvider.class.getName();
+ case SYSTEM_PROPERTIES:
+ return SystemPropertyCredentialsProvider.class.getName();
+ case WEB_IDENTITY:
+ return WebIdentityTokenFileCredentialsProvider.class.getName();
+ case CONTAINER:
+ return ContainerCredentialsProvider.class.getName();
+ case INSTANCE_PROFILE:
+ return InstanceProfileCredentialsProvider.class.getName();
+ case ANONYMOUS:
+ return AnonymousCredentialsProvider.class.getName();
+ case DEFAULT:
+ // For Iceberg REST, use AWS SDK native
DefaultCredentialsProvider
+ return
"software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider";
+ default:
+ throw new UnsupportedOperationException(
+ "AWS SDK V2 does not support credentials provider
mode: " + mode);
+ }
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/IcebergAwsAssumeRoleProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/IcebergAwsAssumeRoleProperties.java
new file mode 100644
index 00000000000..afa224511bc
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/IcebergAwsAssumeRoleProperties.java
@@ -0,0 +1,52 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.common;
+
+import org.apache.doris.datasource.property.storage.S3Properties;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.iceberg.aws.AssumeRoleAwsClientFactory;
+import org.apache.iceberg.aws.AwsProperties;
+
+import java.util.Map;
+
+/**
+ * Shared util for putting Iceberg AWS assume-role properties into a map.
+ * Used by Iceberg REST (glue/s3tables signing), S3 Tables catalog, and S3
FileIO.
+ */
+public final class IcebergAwsAssumeRoleProperties {
+
+ private IcebergAwsAssumeRoleProperties() {}
+
+ /**
+ * Puts assume-role related Iceberg client properties into the target map
when roleArn is present.
+ * No-op if roleArn is blank.
+ */
+ public static void putAssumeRoleProperties(Map<String, String> target,
S3Properties s3Properties) {
+ if (StringUtils.isBlank(s3Properties.getS3IAMRole())) {
+ return;
+ }
+ target.put(AwsProperties.CLIENT_FACTORY,
AssumeRoleAwsClientFactory.class.getName());
+ target.put("aws.region", s3Properties.getRegion());
+ target.put(AwsProperties.CLIENT_ASSUME_ROLE_REGION,
s3Properties.getRegion());
+ target.put(AwsProperties.CLIENT_ASSUME_ROLE_ARN,
s3Properties.getS3IAMRole());
+ if (StringUtils.isNotBlank(s3Properties.getS3ExternalId())) {
+ target.put(AwsProperties.CLIENT_ASSUME_ROLE_EXTERNAL_ID,
s3Properties.getS3ExternalId());
+ }
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/IcebergAwsClientCredentialsProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/IcebergAwsClientCredentialsProperties.java
new file mode 100644
index 00000000000..82228e70037
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/IcebergAwsClientCredentialsProperties.java
@@ -0,0 +1,144 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.common;
+
+import org.apache.doris.datasource.property.storage.S3Properties;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.iceberg.aws.AwsClientProperties;
+import org.apache.iceberg.aws.AwsProperties;
+import org.apache.iceberg.aws.s3.S3FileIOProperties;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+
+import java.util.Map;
+
+public final class IcebergAwsClientCredentialsProperties {
+
+ private IcebergAwsClientCredentialsProperties() {}
+
+ public static void putCredentialProviderProperties(Map<String, String>
target, S3Properties s3Properties) {
+ switch (getCredentialType(s3Properties)) {
+ case EXPLICIT:
+ putExplicitRestCredentials(target,
s3Properties.getAccessKey(), s3Properties.getSecretKey(),
+ s3Properties.getSessionToken());
+ return;
+ case ASSUME_ROLE:
+ IcebergAwsAssumeRoleProperties.putAssumeRoleProperties(target,
s3Properties);
+ return;
+ case PROVIDER_CHAIN:
+ putCredentialsProvider(target,
s3Properties.getAwsCredentialsProviderMode());
+ return;
+ default:
+ throw new IllegalStateException("Unsupported Iceberg AWS
credential type");
+ }
+ }
+
+ public static void putCredentialProviderProperties(Map<String, String>
target,
+ String accessKey, String secretKey, String sessionToken,
AwsCredentialsProviderMode providerMode) {
+ if (StringUtils.isNotBlank(accessKey) &&
StringUtils.isNotBlank(secretKey)) {
+ putExplicitRestCredentials(target, accessKey, secretKey,
sessionToken);
+ return;
+ }
+ putCredentialsProvider(target, providerMode);
+ }
+
+ public static void putS3FileIOCredentialProperties(Map<String, String>
target,
+ S3Properties s3Properties) {
+ putS3FileIOProperties(target, s3Properties);
+ switch (getCredentialType(s3Properties)) {
+ case EXPLICIT:
+ return;
+ case ASSUME_ROLE:
+ IcebergAwsAssumeRoleProperties.putAssumeRoleProperties(target,
s3Properties);
+ return;
+ case PROVIDER_CHAIN:
+ putCredentialsProvider(target,
s3Properties.getAwsCredentialsProviderMode());
+ return;
+ default:
+ throw new IllegalStateException("Unsupported Iceberg AWS
credential type");
+ }
+ }
+
+ public static AwsCredentialsProvider
createAwsCredentialsProvider(S3Properties s3Properties,
+ boolean includeAnonymousInDefault) {
+ switch (getCredentialType(s3Properties)) {
+ case EXPLICIT:
+ case ASSUME_ROLE:
+ return s3Properties.getAwsCredentialsProvider();
+ case PROVIDER_CHAIN:
+ return AwsCredentialsProviderFactory.createV2(
+ s3Properties.getAwsCredentialsProviderMode(),
includeAnonymousInDefault);
+ default:
+ throw new IllegalStateException("Unsupported Iceberg AWS
credential type");
+ }
+ }
+
+ private static CredentialType getCredentialType(S3Properties s3Properties)
{
+ if (StringUtils.isNotBlank(s3Properties.getAccessKey())
+ && StringUtils.isNotBlank(s3Properties.getSecretKey())) {
+ return CredentialType.EXPLICIT;
+ }
+ if (StringUtils.isNotBlank(s3Properties.getS3IAMRole())) {
+ return CredentialType.ASSUME_ROLE;
+ }
+ return CredentialType.PROVIDER_CHAIN;
+ }
+
+ private static void putExplicitRestCredentials(Map<String, String> target,
+ String accessKey, String secretKey, String sessionToken) {
+ target.put(AwsProperties.REST_ACCESS_KEY_ID, accessKey);
+ target.put(AwsProperties.REST_SECRET_ACCESS_KEY, secretKey);
+ if (StringUtils.isNotBlank(sessionToken)) {
+ target.put(AwsProperties.REST_SESSION_TOKEN, sessionToken);
+ }
+ }
+
+ private static void putS3FileIOProperties(Map<String, String> target,
+ S3Properties s3Properties) {
+ if (StringUtils.isNotBlank(s3Properties.getEndpoint())) {
+ target.put(S3FileIOProperties.ENDPOINT,
s3Properties.getEndpoint());
+ }
+ if (StringUtils.isNotBlank(s3Properties.getUsePathStyle())) {
+ target.put(S3FileIOProperties.PATH_STYLE_ACCESS,
s3Properties.getUsePathStyle());
+ }
+ if (StringUtils.isNotBlank(s3Properties.getAccessKey())) {
+ target.put(S3FileIOProperties.ACCESS_KEY_ID,
s3Properties.getAccessKey());
+ }
+ if (StringUtils.isNotBlank(s3Properties.getSecretKey())) {
+ target.put(S3FileIOProperties.SECRET_ACCESS_KEY,
s3Properties.getSecretKey());
+ }
+ if (StringUtils.isNotBlank(s3Properties.getSessionToken())) {
+ target.put(S3FileIOProperties.SESSION_TOKEN,
s3Properties.getSessionToken());
+ }
+ }
+
+ private static void putCredentialsProvider(Map<String, String> target,
+ AwsCredentialsProviderMode providerMode) {
+ if (providerMode == null || providerMode ==
AwsCredentialsProviderMode.DEFAULT) {
+ return;
+ }
+ target.put(AwsClientProperties.CLIENT_CREDENTIALS_PROVIDER,
+ AwsCredentialsProviderFactory.getV2ClassName(providerMode));
+ }
+
+ private enum CredentialType {
+ EXPLICIT,
+ ASSUME_ROLE,
+ PROVIDER_CHAIN
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractIcebergProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractIcebergProperties.java
index 9b23e61eb4d..9a3a5ef5d2a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractIcebergProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractIcebergProperties.java
@@ -20,6 +20,7 @@ package org.apache.doris.datasource.property.metastore;
import org.apache.doris.common.security.authentication.ExecutionAuthenticator;
import org.apache.doris.datasource.iceberg.IcebergExternalCatalog;
import org.apache.doris.datasource.metacache.CacheSpec;
+import
org.apache.doris.datasource.property.common.IcebergAwsAssumeRoleProperties;
import
org.apache.doris.datasource.property.storage.AbstractS3CompatibleProperties;
import org.apache.doris.datasource.property.storage.S3Properties;
import org.apache.doris.datasource.property.storage.StorageProperties;
@@ -266,6 +267,10 @@ public abstract class AbstractIcebergProperties extends
MetastoreProperties {
if (StringUtils.isNotBlank(s3Properties.getSessionToken())) {
options.put(S3FileIOProperties.SESSION_TOKEN,
s3Properties.getSessionToken());
}
+ if (s3Properties instanceof S3Properties) {
+ S3Properties awsProperties = (S3Properties) s3Properties;
+ IcebergAwsAssumeRoleProperties.putAssumeRoleProperties(options,
awsProperties);
+ }
}
protected Catalog buildIcebergCatalog(String catalogName, Map<String,
String> options, Configuration conf) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergRestProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergRestProperties.java
index 887e6d49c5e..407c5c58b3e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergRestProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergRestProperties.java
@@ -18,6 +18,9 @@
package org.apache.doris.datasource.property.metastore;
import org.apache.doris.datasource.iceberg.IcebergExternalCatalog;
+import org.apache.doris.datasource.property.common.AwsCredentialsProviderMode;
+import
org.apache.doris.datasource.property.common.IcebergAwsClientCredentialsProperties;
+import org.apache.doris.datasource.property.storage.S3Properties;
import org.apache.doris.datasource.property.storage.StorageProperties;
import org.apache.doris.foundation.property.ConnectorProperty;
import org.apache.doris.foundation.property.ParamRules;
@@ -41,8 +44,11 @@ public class IcebergRestProperties extends
AbstractIcebergProperties {
private static final String PREFIX_PROPERTY = "prefix";
private static final String VENDED_CREDENTIALS_HEADER =
"header.X-Iceberg-Access-Delegation";
private static final String VENDED_CREDENTIALS_VALUE =
"vended-credentials";
+ private static final String ICEBERG_REST_ROLE_ARN =
"iceberg.rest.role_arn";
+ private static final String ICEBERG_REST_EXTERNAL_ID =
"iceberg.rest.external-id";
private Map<String, String> icebergRestCatalogProperties;
+ private S3Properties s3Properties;
@Getter
@ConnectorProperty(names = {"iceberg.rest.uri", "uri"},
@@ -149,6 +155,22 @@ public class IcebergRestProperties extends
AbstractIcebergProperties {
description = "The secret access key for the iceberg rest catalog
service.")
private String icebergRestSecretAccessKey = "";
+ @ConnectorProperty(names = {"iceberg.rest.session-token"},
+ required = false,
+ sensitive = true,
+ description = "The session-token for the iceberg rest catalog
service.")
+ private String icebergRestSessionToken = "";
+
+ @ConnectorProperty(names = {"iceberg.rest.credentials_provider_type"},
+ required = false,
+ description = "The credentials provider type for AWS
authentication. "
+ + "Options are: DEFAULT, INSTANCE_PROFILE, ENV,
SYSTEM_PROPERTIES, "
+ + "WEB_IDENTITY, CONTAINER. "
+ + "If not set, defaults to DEFAULT (provider chain).")
+ private String icebergRestCredentialsProviderType =
AwsCredentialsProviderMode.DEFAULT.name();
+
+ private AwsCredentialsProviderMode icebergRestCredentialsProviderMode;
+
@ConnectorProperty(names = {"iceberg.rest.connection-timeout-ms"},
required = false,
description = "Connection timeout in milliseconds for the REST
catalog HTTP client. Default: 10000 (10s).")
@@ -182,7 +204,12 @@ public class IcebergRestProperties extends
AbstractIcebergProperties {
public void initNormalizeAndCheckProps() {
super.initNormalizeAndCheckProps();
validateSecurityType();
+ icebergRestCredentialsProviderMode =
+
AwsCredentialsProviderMode.fromString(icebergRestCredentialsProviderType);
buildRules().validate();
+ if (shouldUseS3PropertiesForRestCredentials()) {
+ s3Properties = S3Properties.of(origProps);
+ }
initIcebergRestCatalogProperties();
}
@@ -218,17 +245,32 @@ public class IcebergRestProperties extends
AbstractIcebergProperties {
}
}
- // Check for glue rest catalog specific properties
+ // When signing-name is glue or s3tables: require signing-region and
sigv4-enabled
rules.requireIf(icebergRestSigningName, "glue",
- new String[] {icebergRestSigningRegion,
- icebergRestAccessKeyId,
- icebergRestSecretAccessKey,
- icebergRestSigV4Enabled},
- "Rest Catalog requires signing-region, access-key-id,
secret-access-key "
- + "and sigv4-enabled set to true when signing-name is
glue");
+ new String[] {icebergRestSigningRegion,
icebergRestSigV4Enabled},
+ "Rest Catalog requires signing-region and sigv4-enabled set to
true when signing-name is glue");
+ rules.requireIf(icebergRestSigningName, "s3tables",
+ new String[] {icebergRestSigningRegion,
icebergRestSigV4Enabled},
+ "Rest Catalog requires signing-region and sigv4-enabled set to
true when signing-name is s3tables");
+
+ rejectUnsupportedAwsAssumeRoleProperty(ICEBERG_REST_ROLE_ARN);
+ rejectUnsupportedAwsAssumeRoleProperty(ICEBERG_REST_EXTERNAL_ID);
+
+ // access-key-id and secret-access-key must be set together when
either is set
+ rules.requireTogether(new String[] {icebergRestAccessKeyId,
icebergRestSecretAccessKey},
+ "iceberg.rest.access-key-id and iceberg.rest.secret-access-key
must be set together");
+
return rules;
}
+ private void rejectUnsupportedAwsAssumeRoleProperty(String propertyName) {
+ if (Strings.isNotBlank(origProps.get(propertyName))) {
+ throw new IllegalArgumentException(propertyName + " is not
supported for Iceberg REST catalog. "
+ + "Use iceberg.rest.access-key-id and
iceberg.rest.secret-access-key, "
+ + "or iceberg.rest.credentials_provider_type instead");
+ }
+ }
+
private void initIcebergRestCatalogProperties() {
icebergRestCatalogProperties = new HashMap<>();
// Core catalog properties
@@ -299,12 +341,24 @@ public class IcebergRestProperties extends
AbstractIcebergProperties {
// signing-name is case sensible, do not use lowercase()
icebergRestCatalogProperties.put("rest.signing-name",
icebergRestSigningName);
icebergRestCatalogProperties.put("rest.sigv4-enabled",
icebergRestSigV4Enabled);
- icebergRestCatalogProperties.put("rest.access-key-id",
icebergRestAccessKeyId);
- icebergRestCatalogProperties.put("rest.secret-access-key",
icebergRestSecretAccessKey);
icebergRestCatalogProperties.put("rest.signing-region",
icebergRestSigningRegion);
+
+ if (shouldUseS3PropertiesForRestCredentials()) {
+
IcebergAwsClientCredentialsProperties.putCredentialProviderProperties(
+ icebergRestCatalogProperties, s3Properties);
+ } else {
+
IcebergAwsClientCredentialsProperties.putCredentialProviderProperties(
+ icebergRestCatalogProperties, icebergRestAccessKeyId,
+ icebergRestSecretAccessKey, icebergRestSessionToken,
icebergRestCredentialsProviderMode);
+ }
}
}
+ private boolean shouldUseS3PropertiesForRestCredentials() {
+ return "glue".equals(icebergRestSigningName)
+ || "s3tables".equals(icebergRestSigningName);
+ }
+
public Map<String, String> getIcebergRestCatalogProperties() {
return Collections.unmodifiableMap(icebergRestCatalogProperties);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergS3TablesMetaStoreProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergS3TablesMetaStoreProperties.java
index 58edd445f23..33147cf4da8 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergS3TablesMetaStoreProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergS3TablesMetaStoreProperties.java
@@ -18,14 +18,22 @@
package org.apache.doris.datasource.property.metastore;
import org.apache.doris.datasource.iceberg.IcebergExternalCatalog;
-import
org.apache.doris.datasource.iceberg.s3tables.CustomAwsCredentialsProvider;
+import
org.apache.doris.datasource.property.common.IcebergAwsClientCredentialsProperties;
import org.apache.doris.datasource.property.storage.S3Properties;
import org.apache.doris.datasource.property.storage.StorageProperties;
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.apache.iceberg.aws.AwsClientProperties;
import org.apache.iceberg.catalog.Catalog;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3tables.S3TablesClient;
+import software.amazon.awssdk.services.s3tables.S3TablesClientBuilder;
import software.amazon.s3tables.iceberg.S3TablesCatalog;
+import software.amazon.s3tables.iceberg.S3TablesProperties;
+import software.amazon.s3tables.iceberg.imports.HttpClientProperties;
+import java.net.URI;
import java.util.List;
import java.util.Map;
@@ -52,9 +60,10 @@ public class IcebergS3TablesMetaStoreProperties extends
AbstractIcebergPropertie
public Catalog initCatalog(String catalogName, Map<String, String>
catalogProps,
List<StorageProperties> storagePropertiesList) {
buildS3CatalogProperties(catalogProps);
+ S3TablesClient client = buildS3TablesClient(catalogProps);
S3TablesCatalog catalog = new S3TablesCatalog();
try {
- catalog.initialize(catalogName, catalogProps);
+ catalog.initialize(catalogName, catalogProps, client);
return catalog;
} catch (Exception e) {
throw new RuntimeException("Failed to initialize S3TablesCatalog
for Iceberg. "
@@ -64,10 +73,20 @@ public class IcebergS3TablesMetaStoreProperties extends
AbstractIcebergPropertie
}
private void buildS3CatalogProperties(Map<String, String> props) {
- props.put("client.credentials-provider",
CustomAwsCredentialsProvider.class.getName());
- props.put("client.credentials-provider.s3.access-key-id",
s3Properties.getAccessKey());
- props.put("client.credentials-provider.s3.secret-access-key",
s3Properties.getSecretKey());
- props.put("client.credentials-provider.s3.session-token",
s3Properties.getSessionToken());
- props.put("client.region", s3Properties.getRegion());
+ props.put(AwsClientProperties.CLIENT_REGION, s3Properties.getRegion());
+
IcebergAwsClientCredentialsProperties.putS3FileIOCredentialProperties(props,
s3Properties);
+ }
+
+ private S3TablesClient buildS3TablesClient(Map<String, String> props) {
+ S3TablesClientBuilder builder = S3TablesClient.builder()
+ .region(Region.of(s3Properties.getRegion()))
+
.credentialsProvider(IcebergAwsClientCredentialsProperties.createAwsCredentialsProvider(
+ s3Properties, false));
+ String s3TablesEndpoint =
props.get(S3TablesProperties.S3TABLES_ENDPOINT);
+ if (StringUtils.isNotBlank(s3TablesEndpoint)) {
+ builder.endpointOverride(URI.create(s3TablesEndpoint));
+ }
+ new HttpClientProperties(props).applyHttpClientConfigurations(builder);
+ return builder.build();
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
index 9b2aa2a8c11..db4608be2d0 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
@@ -58,6 +58,11 @@ public class S3Properties extends
AbstractS3CompatibleProperties {
private static final Logger LOG = LogManager.getLogger(S3Properties.class);
public static final String USE_PATH_STYLE = "use_path_style";
+ public static final String ENDPOINT = "s3.endpoint";
+ public static final String REGION = "s3.region";
+ public static final String ROLE_ARN = "s3.role_arn";
+ public static final String EXTERNAL_ID = "s3.external_id";
+ public static final String CREDENTIALS_PROVIDER_TYPE =
"s3.credentials_provider_type";
private static final String[] ENDPOINT_NAMES_FOR_GUESSING = {
"s3.endpoint", "AWS_ENDPOINT", "endpoint", "ENDPOINT",
"aws.endpoint", "glue.endpoint",
@@ -65,12 +70,13 @@ public class S3Properties extends
AbstractS3CompatibleProperties {
};
private static final String[] REGION_NAMES_FOR_GUESSING = {
- "s3.region", "glue.region", "aws.glue.region",
"iceberg.rest.signing-region", "client.region"
+ "s3.region", "glue.region", "aws.glue.region",
"iceberg.rest.signing-region",
+ "rest.signing-region", "client.region"
};
@Setter
@Getter
- @ConnectorProperty(names = {"s3.endpoint", "AWS_ENDPOINT", "endpoint",
"ENDPOINT", "aws.endpoint", "glue.endpoint",
+ @ConnectorProperty(names = {ENDPOINT, "AWS_ENDPOINT", "endpoint",
"ENDPOINT", "aws.endpoint", "glue.endpoint",
"aws.glue.endpoint"},
required = false,
description = "The endpoint of S3.")
@@ -78,8 +84,8 @@ public class S3Properties extends
AbstractS3CompatibleProperties {
@Setter
@Getter
- @ConnectorProperty(names = {"s3.region", "AWS_REGION", "region", "REGION",
"aws.region", "glue.region",
- "aws.glue.region", "iceberg.rest.signing-region", "client.region"},
+ @ConnectorProperty(names = {REGION, "AWS_REGION", "region", "REGION",
"aws.region", "glue.region",
+ "aws.glue.region", "iceberg.rest.signing-region",
"rest.signing-region", "client.region"},
required = false,
isRegionField = true,
description = "The region of S3.")
@@ -104,7 +110,7 @@ public class S3Properties extends
AbstractS3CompatibleProperties {
protected String secretKey = "";
@Getter
- @ConnectorProperty(names = {"s3.session_token", "session_token",
"s3.session-token"},
+ @ConnectorProperty(names = {"s3.session_token", "session_token",
"s3.session-token", "iceberg.rest.session-token"},
required = false,
description = "The session token of S3.")
protected String sessionToken = "";
@@ -161,17 +167,19 @@ public class S3Properties extends
AbstractS3CompatibleProperties {
protected String s3StsRegion = "";
@Getter
- @ConnectorProperty(names = {"s3.role_arn", "AWS_ROLE_ARN",
"glue.role_arn"},
+ @ConnectorProperty(names = {ROLE_ARN, "AWS_ROLE_ARN", "glue.role_arn"},
required = false,
description = "The iam role of S3.")
protected String s3IAMRole = "";
- @ConnectorProperty(names = {"s3.external_id", "AWS_EXTERNAL_ID",
"glue.external_id"},
+ @Getter
+ @ConnectorProperty(names = {EXTERNAL_ID, "AWS_EXTERNAL_ID",
"glue.external_id"},
required = false,
description = "The external id of S3.")
protected String s3ExternalId = "";
- @ConnectorProperty(names = {"s3.credentials_provider_type",
"glue.credentials_provider_type"},
+ @ConnectorProperty(names = {CREDENTIALS_PROVIDER_TYPE,
"glue.credentials_provider_type",
+ "iceberg.rest.credentials_provider_type"},
required = false,
description = "The credentials provider type of S3. "
+ "Options are: DEFAULT, ASSUME_ROLE, ENVIRONMENT,
SYSTEM_PROPERTIES, "
@@ -179,6 +187,7 @@ public class S3Properties extends
AbstractS3CompatibleProperties {
+ "If not set, it will use the default provider chain of
AWS SDK.")
protected String awsCredentialsProviderType =
AwsCredentialsProviderMode.DEFAULT.name();
+ @Getter
private AwsCredentialsProviderMode awsCredentialsProviderMode;
public static S3Properties of(Map<String, String> properties) {
@@ -429,9 +438,7 @@ public class S3Properties extends
AbstractS3CompatibleProperties {
public static final String S3_PREFIX = "s3.";
- public static final String ENDPOINT = "s3.endpoint";
public static final String EXTERNAL_ENDPOINT = "s3.external_endpoint";
- public static final String REGION = "s3.region";
public static final String ACCESS_KEY = "s3.access_key";
public static final String SECRET_KEY = "s3.secret_key";
public static final String SESSION_TOKEN = "s3.session_token";
@@ -439,8 +446,6 @@ public class S3Properties extends
AbstractS3CompatibleProperties {
public static final String REQUEST_TIMEOUT_MS =
"s3.connection.request.timeout";
public static final String CONNECTION_TIMEOUT_MS = "s3.connection.timeout";
- public static final String ROLE_ARN = "s3.role_arn";
- public static final String EXTERNAL_ID = "s3.external_id";
public static final String ROOT_PATH = "s3.root.path";
public static final String BUCKET = "s3.bucket";
public static final String VALIDITY_CHECK = "s3_validity_check";
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergRestPropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergRestPropertiesTest.java
index faf6f95d3b1..47ce0669a6c 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergRestPropertiesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergRestPropertiesTest.java
@@ -357,6 +357,8 @@ public class IcebergRestPropertiesTest {
Assertions.assertFalse(catalogProps.containsKey("rest.secret-access-key"));
Assertions.assertFalse(catalogProps.containsKey("rest.sigv4-enabled"));
props.put("iceberg.rest.signing-name", "custom-service");
+ props.put("iceberg.rest.access-key-id", "AKIAIOSFODNN7EXAMPLE");
+ props.put("iceberg.rest.secret-access-key",
"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY");
restProps = new IcebergRestProperties(props);
restProps.initNormalizeAndCheckProps(); // Should not throw
catalogProps = restProps.getIcebergRestCatalogProperties();
@@ -437,6 +439,307 @@ public class IcebergRestPropertiesTest {
|| errorMessage.contains("sigv4-enabled"));
}
+ @Test
+ public void testS3TablesSigningNameValidWithAccessKeyAndSecretKey() {
+ Map<String, String> props = new HashMap<>();
+ props.put("iceberg.rest.uri", "http://localhost:8080";);
+ props.put("iceberg.rest.signing-name", "s3tables");
+ props.put("iceberg.rest.signing-region", "us-east-1");
+ props.put("iceberg.rest.access-key-id", "AKIAIOSFODNN7EXAMPLE");
+ props.put("iceberg.rest.secret-access-key",
"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY");
+ props.put("iceberg.rest.sigv4-enabled", "true");
+
+ IcebergRestProperties restProps = new IcebergRestProperties(props);
+ Assertions.assertDoesNotThrow(restProps::initNormalizeAndCheckProps);
+
+ Map<String, String> catalogProps =
restProps.getIcebergRestCatalogProperties();
+ Assertions.assertEquals("s3tables",
catalogProps.get("rest.signing-name"));
+ Assertions.assertEquals("us-east-1",
catalogProps.get("rest.signing-region"));
+ Assertions.assertEquals("AKIAIOSFODNN7EXAMPLE",
catalogProps.get("rest.access-key-id"));
+ Assertions.assertEquals("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+ catalogProps.get("rest.secret-access-key"));
+ }
+
+ @Test
+ public void testS3TablesSigningNameCaseInsensitive() {
+ Map<String, String> props = new HashMap<>();
+ props.put("iceberg.rest.uri", "http://localhost:8080";);
+ props.put("iceberg.rest.signing-name", "S3TABLES");
+ props.put("iceberg.rest.signing-region", "us-west-2");
+ props.put("iceberg.rest.access-key-id", "AKIAIOSFODNN7EXAMPLE");
+ props.put("iceberg.rest.secret-access-key",
"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY");
+ props.put("iceberg.rest.sigv4-enabled", "true");
+
+ IcebergRestProperties restProps = new IcebergRestProperties(props);
+ Assertions.assertDoesNotThrow(restProps::initNormalizeAndCheckProps);
+ Assertions.assertEquals("S3TABLES",
restProps.getIcebergRestCatalogProperties().get("rest.signing-name"));
+ }
+
+ @Test
+ public void testGlueSigningNameWithIamRoleFails() {
+ Map<String, String> props = new HashMap<>();
+ props.put("iceberg.rest.uri", "http://localhost:8080";);
+ props.put("iceberg.rest.signing-name", "glue");
+ props.put("iceberg.rest.signing-region", "us-east-1");
+ props.put("iceberg.rest.role_arn",
"arn:aws:iam::123456789012:role/MyGlueRole");
+ props.put("iceberg.rest.sigv4-enabled", "true");
+
+ IcebergRestProperties restProps = new IcebergRestProperties(props);
+ IllegalArgumentException e =
Assertions.assertThrows(IllegalArgumentException.class,
+ restProps::initNormalizeAndCheckProps);
+
Assertions.assertTrue(e.getMessage().contains("iceberg.rest.role_arn"));
+ }
+
+ @Test
+ public void testS3TablesSigningNameWithIamRoleFails() {
+ Map<String, String> props = new HashMap<>();
+ props.put("iceberg.rest.uri", "http://localhost:8080";);
+ props.put("iceberg.rest.signing-name", "s3tables");
+ props.put("iceberg.rest.signing-region", "us-west-2");
+ props.put("iceberg.rest.role_arn",
"arn:aws:iam::999999999999:role/S3TablesRole");
+ props.put("iceberg.rest.sigv4-enabled", "true");
+
+ IcebergRestProperties restProps = new IcebergRestProperties(props);
+ IllegalArgumentException e =
Assertions.assertThrows(IllegalArgumentException.class,
+ restProps::initNormalizeAndCheckProps);
+
Assertions.assertTrue(e.getMessage().contains("iceberg.rest.role_arn"));
+ }
+
+ @Test
+ public void testGlueSigningNameWithDefaultCredentialsProvider() {
+ Map<String, String> props = new HashMap<>();
+ props.put("iceberg.rest.uri", "http://localhost:8080";);
+ props.put("iceberg.rest.signing-name", "glue");
+ props.put("iceberg.rest.signing-region", "us-east-1");
+ props.put("iceberg.rest.sigv4-enabled", "true");
+
+ IcebergRestProperties restProps = new IcebergRestProperties(props);
+ Assertions.assertDoesNotThrow(restProps::initNormalizeAndCheckProps);
+
+ Map<String, String> catalogProps =
restProps.getIcebergRestCatalogProperties();
+
Assertions.assertFalse(catalogProps.containsKey("client.credentials-provider"));
+ }
+
+ @Test
+ public void testS3TablesSigningNameWithDefaultCredentialsProvider() {
+ Map<String, String> props = new HashMap<>();
+ props.put("iceberg.rest.uri", "http://localhost:8080";);
+ props.put("iceberg.rest.signing-name", "s3tables");
+ props.put("iceberg.rest.signing-region", "us-east-1");
+ props.put("iceberg.rest.sigv4-enabled", "true");
+ // No credentials, should use DEFAULT provider
+
+ IcebergRestProperties restProps = new IcebergRestProperties(props);
+ Assertions.assertDoesNotThrow(restProps::initNormalizeAndCheckProps);
+
+ Map<String, String> catalogProps =
restProps.getIcebergRestCatalogProperties();
+
Assertions.assertFalse(catalogProps.containsKey("client.credentials-provider"));
+ }
+
+ @Test
+ public void testS3TablesSigningNameMissingSigningRegionFails() {
+ Map<String, String> props = new HashMap<>();
+ props.put("iceberg.rest.uri", "http://localhost:8080";);
+ props.put("iceberg.rest.signing-name", "s3tables");
+ props.put("iceberg.rest.access-key-id", "AKIAIOSFODNN7EXAMPLE");
+ props.put("iceberg.rest.secret-access-key",
"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY");
+ props.put("iceberg.rest.sigv4-enabled", "true");
+
+ IcebergRestProperties restProps = new IcebergRestProperties(props);
+ IllegalArgumentException e =
Assertions.assertThrows(IllegalArgumentException.class,
+ restProps::initNormalizeAndCheckProps);
+ Assertions.assertTrue(e.getMessage().contains("signing-region") &&
e.getMessage().contains("s3tables"));
+ }
+
+ @Test
+ public void testAccessKeyAndSecretKeyMustBeSetTogether() {
+ Map<String, String> props1 = new HashMap<>();
+ props1.put("iceberg.rest.uri", "http://localhost:8080";);
+ props1.put("iceberg.rest.signing-name", "glue");
+ props1.put("iceberg.rest.signing-region", "us-east-1");
+ props1.put("iceberg.rest.access-key-id", "AKIAIOSFODNN7EXAMPLE");
+ props1.put("iceberg.rest.sigv4-enabled", "true");
+
+ IcebergRestProperties restProps1 = new IcebergRestProperties(props1);
+ IllegalArgumentException e1 =
Assertions.assertThrows(IllegalArgumentException.class,
+ restProps1::initNormalizeAndCheckProps);
+ Assertions.assertTrue(e1.getMessage().contains("access-key-id")
+ && e1.getMessage().contains("secret-access-key"));
+
+ Map<String, String> props2 = new HashMap<>();
+ props2.put("iceberg.rest.uri", "http://localhost:8080";);
+ props2.put("iceberg.rest.signing-name", "glue");
+ props2.put("iceberg.rest.signing-region", "us-east-1");
+ props2.put("iceberg.rest.secret-access-key",
"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY");
+ props2.put("iceberg.rest.sigv4-enabled", "true");
+
+ IcebergRestProperties restProps2 = new IcebergRestProperties(props2);
+ Assertions.assertThrows(IllegalArgumentException.class,
restProps2::initNormalizeAndCheckProps);
+ }
+
+ @Test
+ public void testGlueWithIamRoleAndExternalIdFails() {
+ Map<String, String> props = new HashMap<>();
+ props.put("iceberg.rest.uri", "http://localhost:8080";);
+ props.put("iceberg.rest.signing-name", "glue");
+ props.put("iceberg.rest.signing-region", "us-east-1");
+ props.put("iceberg.rest.role_arn",
"arn:aws:iam::123456789012:role/MyGlueRole");
+ props.put("iceberg.rest.external-id", "external-123");
+ props.put("iceberg.rest.sigv4-enabled", "true");
+
+ IcebergRestProperties restProps = new IcebergRestProperties(props);
+ IllegalArgumentException e =
Assertions.assertThrows(IllegalArgumentException.class,
+ restProps::initNormalizeAndCheckProps);
+
Assertions.assertTrue(e.getMessage().contains("iceberg.rest.role_arn"));
+ }
+
+ @Test
+ public void testGlueWithExternalIdFails() {
+ Map<String, String> props = new HashMap<>();
+ props.put("iceberg.rest.uri", "http://localhost:8080";);
+ props.put("iceberg.rest.signing-name", "glue");
+ props.put("iceberg.rest.signing-region", "us-east-1");
+ props.put("iceberg.rest.external-id", "external-123");
+ props.put("iceberg.rest.sigv4-enabled", "true");
+
+ IcebergRestProperties restProps = new IcebergRestProperties(props);
+ IllegalArgumentException e =
Assertions.assertThrows(IllegalArgumentException.class,
+ restProps::initNormalizeAndCheckProps);
+
Assertions.assertTrue(e.getMessage().contains("iceberg.rest.external-id"));
+ }
+
+ @Test
+ public void testGlueWithCredentialsProviderTypeDefault() {
+ Map<String, String> props = new HashMap<>();
+ props.put("iceberg.rest.uri", "http://localhost:8080";);
+ props.put("iceberg.rest.signing-name", "glue");
+ props.put("iceberg.rest.signing-region", "us-east-1");
+ props.put("iceberg.rest.sigv4-enabled", "true");
+ props.put("iceberg.rest.credentials_provider_type", "DEFAULT");
+
+ IcebergRestProperties restProps = new IcebergRestProperties(props);
+ Assertions.assertDoesNotThrow(restProps::initNormalizeAndCheckProps);
+
+ Map<String, String> catalogProps =
restProps.getIcebergRestCatalogProperties();
+ Assertions.assertEquals("glue", catalogProps.get("rest.signing-name"));
+ Assertions.assertEquals("us-east-1",
catalogProps.get("rest.signing-region"));
+
Assertions.assertFalse(catalogProps.containsKey("client.credentials-provider"));
+ }
+
+ @Test
+ public void testS3TablesWithCredentialsProviderTypeInstanceProfile() {
+ Map<String, String> props = new HashMap<>();
+ props.put("iceberg.rest.uri", "http://localhost:8080";);
+ props.put("iceberg.rest.signing-name", "s3tables");
+ props.put("iceberg.rest.signing-region", "us-west-2");
+ props.put("iceberg.rest.sigv4-enabled", "true");
+ props.put("iceberg.rest.credentials_provider_type",
"INSTANCE_PROFILE");
+
+ IcebergRestProperties restProps = new IcebergRestProperties(props);
+ Assertions.assertDoesNotThrow(restProps::initNormalizeAndCheckProps);
+
+ Map<String, String> catalogProps =
restProps.getIcebergRestCatalogProperties();
+ Assertions.assertEquals("s3tables",
catalogProps.get("rest.signing-name"));
+ Assertions.assertEquals(
+
"software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider",
+ catalogProps.get("client.credentials-provider"));
+ }
+
+ @Test
+ public void testS3TablesWithCredentialsProviderTypeEnvWithoutAccessKey() {
+ Map<String, String> props = new HashMap<>();
+ props.put("iceberg.rest.uri", "http://localhost:8080";);
+ props.put("iceberg.rest.signing-name", "s3tables");
+ props.put("iceberg.rest.signing-region", "us-west-2");
+ props.put("iceberg.rest.sigv4-enabled", "true");
+ props.put("iceberg.rest.credentials_provider_type", "ENV");
+
+ IcebergRestProperties restProps = new IcebergRestProperties(props);
+ Assertions.assertDoesNotThrow(restProps::initNormalizeAndCheckProps);
+
+ Map<String, String> catalogProps =
restProps.getIcebergRestCatalogProperties();
+ Assertions.assertEquals("s3tables",
catalogProps.get("rest.signing-name"));
+ Assertions.assertEquals("us-west-2",
catalogProps.get("rest.signing-region"));
+ Assertions.assertEquals(
+
"software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider",
+ catalogProps.get("client.credentials-provider"));
+ Assertions.assertFalse(catalogProps.containsKey("rest.access-key-id"));
+
Assertions.assertFalse(catalogProps.containsKey("rest.secret-access-key"));
+ }
+
+ @Test
+ public void testGlueWithCredentialsProviderTypeEnv() {
+ Map<String, String> props = new HashMap<>();
+ props.put("iceberg.rest.uri", "http://localhost:8080";);
+ props.put("iceberg.rest.signing-name", "glue");
+ props.put("iceberg.rest.signing-region", "ap-east-1");
+ props.put("iceberg.rest.sigv4-enabled", "true");
+ props.put("iceberg.rest.credentials_provider_type", "ENV");
+
+ IcebergRestProperties restProps = new IcebergRestProperties(props);
+ Assertions.assertDoesNotThrow(restProps::initNormalizeAndCheckProps);
+
+ Map<String, String> catalogProps =
restProps.getIcebergRestCatalogProperties();
+ Assertions.assertEquals(
+
"software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider",
+ catalogProps.get("client.credentials-provider"));
+ }
+
+ @Test
+ public void testAccessKeyPriorityOverCredentialsProviderType() {
+ Map<String, String> props = new HashMap<>();
+ props.put("iceberg.rest.uri", "http://localhost:8080";);
+ props.put("iceberg.rest.signing-name", "glue");
+ props.put("iceberg.rest.signing-region", "us-east-1");
+ props.put("iceberg.rest.sigv4-enabled", "true");
+ props.put("iceberg.rest.access-key-id", "AKIAIOSFODNN7EXAMPLE");
+ props.put("iceberg.rest.secret-access-key",
"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY");
+ props.put("iceberg.rest.credentials_provider_type", "DEFAULT");
+
+ IcebergRestProperties restProps = new IcebergRestProperties(props);
+ Assertions.assertDoesNotThrow(restProps::initNormalizeAndCheckProps);
+
+ Map<String, String> catalogProps =
restProps.getIcebergRestCatalogProperties();
+ Assertions.assertEquals("AKIAIOSFODNN7EXAMPLE",
+ catalogProps.get("rest.access-key-id"));
+ Assertions.assertEquals("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+ catalogProps.get("rest.secret-access-key"));
+
Assertions.assertFalse(catalogProps.containsKey("client.credentials-provider"));
+
Assertions.assertFalse(catalogProps.containsKey("client.credentials-provider.s3.access-key-id"));
+
Assertions.assertFalse(catalogProps.containsKey("client.credentials-provider.s3.secret-access-key"));
+ }
+
+ @Test
+ public void testIamRoleWithCredentialsProviderTypeFails() {
+ Map<String, String> props = new HashMap<>();
+ props.put("iceberg.rest.uri", "http://localhost:8080";);
+ props.put("iceberg.rest.signing-name", "s3tables");
+ props.put("iceberg.rest.signing-region", "us-west-2");
+ props.put("iceberg.rest.sigv4-enabled", "true");
+ props.put("iceberg.rest.role_arn",
"arn:aws:iam::123456789012:role/MyRole");
+ props.put("iceberg.rest.credentials_provider_type",
"INSTANCE_PROFILE");
+
+ IcebergRestProperties restProps = new IcebergRestProperties(props);
+ IllegalArgumentException e =
Assertions.assertThrows(IllegalArgumentException.class,
+ restProps::initNormalizeAndCheckProps);
+
Assertions.assertTrue(e.getMessage().contains("iceberg.rest.role_arn"));
+ }
+
+ @Test
+ public void testNonGlueSigningNameWithoutCredentialsAllowed() {
+ Map<String, String> props = new HashMap<>();
+ props.put("iceberg.rest.uri", "http://localhost:8080";);
+ props.put("iceberg.rest.signing-name", "custom-service");
+ props.put("iceberg.rest.signing-region", "us-east-1");
+ props.put("iceberg.rest.sigv4-enabled", "true");
+
+ IcebergRestProperties restProps = new IcebergRestProperties(props);
+ Assertions.assertDoesNotThrow(restProps::initNormalizeAndCheckProps);
+
+ Map<String, String> catalogProps =
restProps.getIcebergRestCatalogProperties();
+
Assertions.assertFalse(catalogProps.containsKey("client.credentials-provider"));
+ }
+
@Test
public void testToFileIOPropertiesPrefersNonS3Properties() {
// When both S3Properties and OSSProperties exist, OSSProperties
should be chosen
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergS3TablesMetaStorePropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergS3TablesMetaStorePropertiesTest.java
index 423f74856c7..3043ecc02c5 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergS3TablesMetaStorePropertiesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergS3TablesMetaStorePropertiesTest.java
@@ -18,18 +18,61 @@
package org.apache.doris.datasource.property.metastore;
import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.iceberg.IcebergExternalCatalog;
+import
org.apache.doris.datasource.property.common.IcebergAwsClientCredentialsProperties;
+import org.apache.doris.datasource.property.storage.S3Properties;
import org.apache.doris.datasource.property.storage.StorageProperties;
import com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.aws.AssumeRoleAwsClientFactory;
+import org.apache.iceberg.aws.AwsClientProperties;
+import org.apache.iceberg.aws.AwsProperties;
+import org.apache.iceberg.aws.s3.S3FileIOProperties;
import org.apache.iceberg.catalog.Catalog;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import software.amazon.s3tables.iceberg.S3TablesCatalog;
+import java.lang.reflect.Method;
+import java.util.HashMap;
import java.util.Map;
public class IcebergS3TablesMetaStorePropertiesTest {
+ /**
+ * Call private buildS3CatalogProperties to fill catalogProps without
initializing S3TablesCatalog
+ * (which requires warehouse/table bucket ARN and would throw
ValidationException).
+ */
+ private static void
buildS3CatalogProperties(IcebergS3TablesMetaStoreProperties metaProps,
+ Map<String, String> catalogProps) throws Exception {
+ Method m =
IcebergS3TablesMetaStoreProperties.class.getDeclaredMethod("buildS3CatalogProperties",
Map.class);
+ m.setAccessible(true);
+ m.invoke(metaProps, catalogProps);
+ }
+
+ @Test
+ public void s3FileIOCredentialPropertiesUseSharedS3Properties() {
+ Map<String, String> props = new HashMap<>();
+ props.put("s3.region", "us-east-1");
+ props.put("s3.endpoint", "https://s3.us-east-1.amazonaws.com";);
+ props.put("s3.access_key", "AKID");
+ props.put("s3.secret_key", "SECRET");
+ props.put("s3.session_token", "TOKEN");
+ props.put("s3.credentials_provider_type", "INSTANCE_PROFILE");
+
+ Map<String, String> catalogProps = new HashMap<>();
+ IcebergAwsClientCredentialsProperties.putS3FileIOCredentialProperties(
+ catalogProps, S3Properties.of(props));
+
+ Assertions.assertEquals("https://s3.us-east-1.amazonaws.com";,
+ catalogProps.get(S3FileIOProperties.ENDPOINT));
+ Assertions.assertEquals("AKID",
catalogProps.get(S3FileIOProperties.ACCESS_KEY_ID));
+ Assertions.assertEquals("SECRET",
catalogProps.get(S3FileIOProperties.SECRET_ACCESS_KEY));
+ Assertions.assertEquals("TOKEN",
catalogProps.get(S3FileIOProperties.SESSION_TOKEN));
+
Assertions.assertFalse(catalogProps.containsKey(AwsClientProperties.CLIENT_CREDENTIALS_PROVIDER));
+
Assertions.assertFalse(catalogProps.containsKey(AwsProperties.CLIENT_ASSUME_ROLE_ARN));
+ }
+
@Test
public void s3TablesTest() throws UserException {
Map<String, String> baseProps = ImmutableMap.of(
@@ -52,4 +95,233 @@ public class IcebergS3TablesMetaStorePropertiesTest {
Assertions.assertEquals(S3TablesCatalog.class, catalog.getClass());
}
+ @Test
+ public void s3TablesWithIamRole() throws Exception {
+ Map<String, String> props = new HashMap<>();
+ props.put("type", "iceberg");
+ props.put("iceberg.catalog.type", "s3tables");
+ props.put("warehouse", "s3://my-bucket/warehouse");
+ props.put("s3.region", "us-east-1");
+ props.put("s3.role_arn",
"arn:aws:iam::123456789012:role/S3TablesRole");
+ props.put("s3.endpoint", "https://s3.us-east-1.amazonaws.com";);
+
+ IcebergS3TablesMetaStoreProperties metaProps = new
IcebergS3TablesMetaStoreProperties(props);
+ metaProps.initNormalizeAndCheckProps();
+
+ Assertions.assertEquals(IcebergExternalCatalog.ICEBERG_S3_TABLES,
metaProps.getIcebergCatalogType());
+
+ Map<String, String> catalogProps = new HashMap<>();
+ buildS3CatalogProperties(metaProps, catalogProps);
+
+ Assertions.assertEquals(AssumeRoleAwsClientFactory.class.getName(),
+ catalogProps.get(AwsProperties.CLIENT_FACTORY));
+
Assertions.assertFalse(catalogProps.containsKey(S3FileIOProperties.CLIENT_FACTORY));
+ Assertions.assertEquals("arn:aws:iam::123456789012:role/S3TablesRole",
catalogProps.get("client.assume-role.arn"));
+ Assertions.assertEquals("us-east-1",
catalogProps.get("client.assume-role.region"));
+
Assertions.assertFalse(catalogProps.containsKey("client.credentials-provider"));
+
Assertions.assertFalse(catalogProps.containsKey("client.credentials-provider.s3.role_arn"));
+
Assertions.assertFalse(catalogProps.containsKey("client.credentials-provider.s3.region"));
+
Assertions.assertFalse(catalogProps.containsKey("client.credentials-provider.s3.credentials_provider_type"));
+ Assertions.assertFalse(catalogProps.containsKey(
+ "client.credentials-provider.assume-role.arn"));
+ Assertions.assertFalse(catalogProps.containsKey(
+
"client.credentials-provider.assume-role.source-credentials-provider"));
+ Assertions.assertFalse(catalogProps.containsKey(
+
"client.credentials-provider.assume-role.source-provider-type"));
+ Assertions.assertFalse(catalogProps.containsKey(
+ "client.credentials-provider.client.assume-role.arn"));
+ }
+
+ @Test
+ public void s3TablesWithIamRoleAndExternalId() throws Exception {
+ Map<String, String> props = new HashMap<>();
+ props.put("type", "iceberg");
+ props.put("iceberg.catalog.type", "s3tables");
+ props.put("warehouse", "s3://my-bucket/warehouse");
+ props.put("s3.region", "us-west-2");
+ props.put("s3.role_arn", "arn:aws:iam::999999999999:role/MyRole");
+ props.put("s3.external_id", "external-id-123");
+ props.put("s3.endpoint", "https://s3.us-west-2.amazonaws.com";);
+
+ IcebergS3TablesMetaStoreProperties metaProps = new
IcebergS3TablesMetaStoreProperties(props);
+ metaProps.initNormalizeAndCheckProps();
+
+ Map<String, String> catalogProps = new HashMap<>();
+ buildS3CatalogProperties(metaProps, catalogProps);
+
+ Assertions.assertEquals("arn:aws:iam::999999999999:role/MyRole",
catalogProps.get("client.assume-role.arn"));
+ Assertions.assertEquals("external-id-123",
catalogProps.get("client.assume-role.external-id"));
+
Assertions.assertFalse(catalogProps.containsKey("client.credentials-provider.s3.external_id"));
+ Assertions.assertFalse(catalogProps.containsKey(
+ "client.credentials-provider.assume-role.external-id"));
+ }
+
+ @Test
+ public void s3TablesWithAccessKeyPreferOverIamRole() throws Exception {
+ Map<String, String> props = new HashMap<>();
+ props.put("type", "iceberg");
+ props.put("iceberg.catalog.type", "s3tables");
+ props.put("warehouse", "s3://my-bucket/warehouse");
+ props.put("s3.region", "us-east-1");
+ props.put("s3.access_key", "AKID");
+ props.put("s3.secret_key", "SECRET");
+ props.put("s3.role_arn", "arn:aws:iam::123456789012:role/Role");
+ props.put("s3.endpoint", "https://s3.us-east-1.amazonaws.com";);
+
+ IcebergS3TablesMetaStoreProperties metaProps = new
IcebergS3TablesMetaStoreProperties(props);
+ metaProps.initNormalizeAndCheckProps();
+
+ Map<String, String> catalogProps = new HashMap<>();
+ buildS3CatalogProperties(metaProps, catalogProps);
+
+
Assertions.assertFalse(catalogProps.containsKey("client.credentials-provider"));
+ Assertions.assertEquals("AKID",
catalogProps.get(S3FileIOProperties.ACCESS_KEY_ID));
+ Assertions.assertEquals("SECRET",
catalogProps.get(S3FileIOProperties.SECRET_ACCESS_KEY));
+ Assertions.assertNull(catalogProps.get("client.assume-role.arn"));
+ }
+
+ // --- UT for credentials_provider_type support ---
+
+ @Test
+ public void s3TablesWithCredentialsProviderTypeDefault() throws Exception {
+ Map<String, String> props = new HashMap<>();
+ props.put("type", "iceberg");
+ props.put("iceberg.catalog.type", "s3tables");
+ props.put("warehouse", "s3://my-bucket/warehouse");
+ props.put("s3.region", "us-west-2");
+ props.put("s3.endpoint", "https://s3.us-west-2.amazonaws.com";);
+ props.put("s3.credentials_provider_type", "DEFAULT");
+
+ IcebergS3TablesMetaStoreProperties metaProps = new
IcebergS3TablesMetaStoreProperties(props);
+ metaProps.initNormalizeAndCheckProps();
+
+ Map<String, String> catalogProps = new HashMap<>();
+ buildS3CatalogProperties(metaProps, catalogProps);
+
+ Assertions.assertEquals("us-west-2",
catalogProps.get("client.region"));
+
Assertions.assertFalse(catalogProps.containsKey("client.credentials-provider"));
+ }
+
+ @Test
+ public void s3TablesWithCredentialsProviderTypeInstanceProfile() throws
Exception {
+ Map<String, String> props = new HashMap<>();
+ props.put("type", "iceberg");
+ props.put("iceberg.catalog.type", "s3tables");
+ props.put("warehouse", "s3://my-bucket/warehouse");
+ props.put("s3.region", "ap-east-1");
+ props.put("s3.endpoint", "https://s3.ap-east-1.amazonaws.com";);
+ props.put("s3.credentials_provider_type", "INSTANCE_PROFILE");
+
+ IcebergS3TablesMetaStoreProperties metaProps = new
IcebergS3TablesMetaStoreProperties(props);
+ metaProps.initNormalizeAndCheckProps();
+
+ Map<String, String> catalogProps = new HashMap<>();
+ buildS3CatalogProperties(metaProps, catalogProps);
+
+
Assertions.assertEquals("software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider",
+ catalogProps.get("client.credentials-provider"));
+
Assertions.assertFalse(catalogProps.containsKey("client.credentials-provider.s3.credentials_provider_type"));
+ }
+
+ @Test
+ public void s3TablesWithCredentialsProviderTypeEnv() throws Exception {
+ Map<String, String> props = new HashMap<>();
+ props.put("type", "iceberg");
+ props.put("iceberg.catalog.type", "s3tables");
+ props.put("warehouse", "s3://my-bucket/warehouse");
+ props.put("s3.region", "us-east-1");
+ props.put("s3.endpoint", "https://s3.us-east-1.amazonaws.com";);
+ props.put("s3.credentials_provider_type", "ENV");
+
+ IcebergS3TablesMetaStoreProperties metaProps = new
IcebergS3TablesMetaStoreProperties(props);
+ metaProps.initNormalizeAndCheckProps();
+
+ Map<String, String> catalogProps = new HashMap<>();
+ buildS3CatalogProperties(metaProps, catalogProps);
+
+
Assertions.assertEquals("software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider",
+ catalogProps.get("client.credentials-provider"));
+
Assertions.assertFalse(catalogProps.containsKey("client.credentials-provider.s3.credentials_provider_type"));
+ }
+
+ @Test
+ public void s3TablesAccessKeyPriorityOverCredentialsProviderType() throws
Exception {
+ Map<String, String> props = new HashMap<>();
+ props.put("type", "iceberg");
+ props.put("iceberg.catalog.type", "s3tables");
+ props.put("warehouse", "s3://my-bucket/warehouse");
+ props.put("s3.region", "us-west-2");
+ props.put("s3.access_key", "AKIAIOSFODNN7EXAMPLE");
+ props.put("s3.secret_key", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY");
+ props.put("s3.credentials_provider_type", "INSTANCE_PROFILE");
+ props.put("s3.endpoint", "https://s3.us-west-2.amazonaws.com";);
+
+ IcebergS3TablesMetaStoreProperties metaProps = new
IcebergS3TablesMetaStoreProperties(props);
+ metaProps.initNormalizeAndCheckProps();
+
+ Map<String, String> catalogProps = new HashMap<>();
+ buildS3CatalogProperties(metaProps, catalogProps);
+
+ // Access key should take priority
+ Assertions.assertEquals("AKIAIOSFODNN7EXAMPLE",
catalogProps.get(S3FileIOProperties.ACCESS_KEY_ID));
+ Assertions.assertEquals("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+ catalogProps.get(S3FileIOProperties.SECRET_ACCESS_KEY));
+
Assertions.assertFalse(catalogProps.containsKey("client.credentials-provider"));
+ }
+
+ @Test
+ public void s3TablesIamRolePriorityOverCredentialsProviderType() throws
Exception {
+ Map<String, String> props = new HashMap<>();
+ props.put("type", "iceberg");
+ props.put("iceberg.catalog.type", "s3tables");
+ props.put("warehouse", "s3://my-bucket/warehouse");
+ props.put("s3.region", "us-east-1");
+ props.put("s3.role_arn",
"arn:aws:iam::123456789012:role/S3TablesRole");
+ props.put("s3.credentials_provider_type", "INSTANCE_PROFILE");
+ props.put("s3.endpoint", "https://s3.us-east-1.amazonaws.com";);
+
+ IcebergS3TablesMetaStoreProperties metaProps = new
IcebergS3TablesMetaStoreProperties(props);
+ metaProps.initNormalizeAndCheckProps();
+
+ Map<String, String> catalogProps = new HashMap<>();
+ buildS3CatalogProperties(metaProps, catalogProps);
+
+ // IAM Role should take priority
+ Assertions.assertEquals("arn:aws:iam::123456789012:role/S3TablesRole",
+ catalogProps.get("client.assume-role.arn"));
+ Assertions.assertEquals(AssumeRoleAwsClientFactory.class.getName(),
+ catalogProps.get(AwsProperties.CLIENT_FACTORY));
+
Assertions.assertFalse(catalogProps.containsKey(S3FileIOProperties.CLIENT_FACTORY));
+
Assertions.assertFalse(catalogProps.containsKey("client.credentials-provider"));
+ Assertions.assertFalse(catalogProps.containsKey(
+ "client.credentials-provider.s3.credentials_provider_type"));
+
Assertions.assertFalse(catalogProps.containsKey("client.credentials-provider.s3.role_arn"));
+ Assertions.assertFalse(catalogProps.containsKey(
+
"client.credentials-provider.assume-role.source-credentials-provider"));
+ Assertions.assertFalse(catalogProps.containsKey(
+
"client.credentials-provider.assume-role.source-provider-type"));
+ Assertions.assertFalse(catalogProps.containsKey(
+ "client.credentials-provider.client.credentials-provider"));
+ }
+
+ @Test
+ public void s3TablesDefaultCredentialsProviderTypeWhenNothingSet() throws
Exception {
+ Map<String, String> props = new HashMap<>();
+ props.put("type", "iceberg");
+ props.put("iceberg.catalog.type", "s3tables");
+ props.put("warehouse", "s3://my-bucket/warehouse");
+ props.put("s3.region", "us-west-2");
+ props.put("s3.endpoint", "https://s3.us-west-2.amazonaws.com";);
+ // Not setting any credentials or credentials_provider_type
+ // S3Properties defaults to DEFAULT mode
+
+ IcebergS3TablesMetaStoreProperties metaProps = new
IcebergS3TablesMetaStoreProperties(props);
+ metaProps.initNormalizeAndCheckProps();
+
+ Map<String, String> catalogProps = new HashMap<>();
+ buildS3CatalogProperties(metaProps, catalogProps);
+
+ // Let AWS SDK use its default provider chain when no credentials are
provided.
+
Assertions.assertFalse(catalogProps.containsKey("client.credentials-provider"));
+ }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
index 481286f9886..abe52d64cc4 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
@@ -247,6 +247,19 @@ public class S3PropertiesTest {
Assertions.assertEquals("arn:aws:iam::123456789012:role/MyTestRole",
backendProperties.get("AWS_ROLE_ARN"));
}
+ @Test
+ public void testS3PropertiesIgnoreIcebergRestIamRoleAliases() throws
UserException {
+ origProps.put("s3.endpoint", "s3.us-west-2.amazonaws.com");
+ origProps.put("iceberg.rest.role_arn",
"arn:aws:iam::123456789012:role/MyTestRole");
+ origProps.put("iceberg.rest.external-id", "external-123");
+
+ S3Properties s3Props = (S3Properties)
StorageProperties.createPrimary(origProps);
+ Map<String, String> backendProperties =
s3Props.getBackendConfigProperties();
+
+ Assertions.assertNull(backendProperties.get("AWS_ROLE_ARN"));
+ Assertions.assertNull(backendProperties.get("AWS_EXTERNAL_ID"));
+ }
+
@Test
public void testGetAwsCredentialsProviderWithIamRoleAndExternalId() {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/s3tables/S3TablesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/s3tables/S3TablesTest.java
index 576033785ed..4232de6f1c2 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/s3tables/S3TablesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/s3tables/S3TablesTest.java
@@ -17,8 +17,6 @@
package org.apache.doris.datasource.s3tables;
-import
org.apache.doris.datasource.iceberg.s3tables.CustomAwsCredentialsProvider;
-
import org.apache.iceberg.FileScanTask;
import org.apache.iceberg.Table;
import org.apache.iceberg.TableScan;
@@ -39,14 +37,7 @@ public class S3TablesTest {
S3TablesCatalog s3TablesCatalog = new S3TablesCatalog();
Map<String, String> s3Properties = new HashMap<>();
- // ak, sk
- String accessKeyId = "";
- String secretKey = "";
-
s3Properties.put("client.region", "us-east-1");
- s3Properties.put("client.credentials-provider",
CustomAwsCredentialsProvider.class.getName());
- s3Properties.put("client.credentials-provider.s3.access-key-id",
accessKeyId);
- s3Properties.put("client.credentials-provider.s3.secret-access-key",
secretKey);
String warehouse =
"arn:aws:s3tables:us-east-1:169698404049:bucket/yy-s3-table-bucket";
s3Properties.put("warehouse", warehouse);
diff --git
a/regression-test/suites/aws_iam_role_p0/test_iceberg_s3tables_catalog_credentials_provider.groovy
b/regression-test/suites/aws_iam_role_p0/test_iceberg_s3tables_catalog_credentials_provider.groovy
new file mode 100644
index 00000000000..bb0d67327d1
--- /dev/null
+++
b/regression-test/suites/aws_iam_role_p0/test_iceberg_s3tables_catalog_credentials_provider.groovy
@@ -0,0 +1,105 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import com.google.common.base.Strings;
+
+suite("test_iceberg_s3tables_catalog_credentials_provider") {
+
+ List<String> requiredConfigNames = [
+ "icebergS3TablesCatalog1Props",
+ "icebergS3TablesCatalog2Props",
+ "icebergS3TablesCatalog3Props",
+ "icebergS3TablesPartitionDbTableName",
+ "icebergS3TablesNonPartitionDbTableName",
+ "icebergS3TablesExpectCounts"
+ ]
+ if (requiredConfigNames.any {
Strings.isNullOrEmpty(context.config.otherConfigs.get(it)) }) {
+ return
+ }
+
+ sql """ ADMIN SET FRONTEND CONFIG
("aws_credentials_provider_version"="v2"); """
+
+ String catalog1Props =
context.config.otherConfigs.get("icebergS3TablesCatalog1Props")
+ String catalog2Props =
context.config.otherConfigs.get("icebergS3TablesCatalog2Props")
+ String catalog3Props =
context.config.otherConfigs.get("icebergS3TablesCatalog3Props")
+ String partitionDbTableName =
context.config.otherConfigs.get("icebergS3TablesPartitionDbTableName")
+ String nonPartitionDbTableName =
context.config.otherConfigs.get("icebergS3TablesNonPartitionDbTableName")
+ String expectCounts =
context.config.otherConfigs.get("icebergS3TablesExpectCounts")
+ String expectedWebIdentityError = "Either the environment variable
AWS_WEB_IDENTITY_TOKEN_FILE or the javaproperty aws.webIdentityTokenFile must
be set."
+
+ def createCatalogAndQueryTables = {
+ catalogProps, catalogName, partitionQueryDbTableName,
nonPartitionQueryDbTableName, expectedCounts ->
+ sql """drop catalog if exists ${catalogName}"""
+ try {
+ sql """
+ create catalog ${catalogName} properties (
+ ${catalogProps}
+ );
+ """
+ def partitionResult = sql """
+ select count(1) from
${catalogName}.${partitionQueryDbTableName};
+ """
+ def nonPartitionResult = sql """
+ select count(1) from
${catalogName}.${nonPartitionQueryDbTableName};
+ """
+ def partitionCount = partitionResult[0][0]
+ def nonPartitionCount = nonPartitionResult[0][0]
+ assertTrue(partitionCount == expectedCounts.toInteger())
+ assertTrue(nonPartitionCount == expectedCounts.toInteger())
+ assertTrue(partitionCount == nonPartitionCount)
+ } finally {
+ sql """drop catalog if exists ${catalogName}"""
+ }
+ }
+
+ def createCatalogAndShowDatabasesException = { catalogProps, catalogName,
expectedMessage ->
+ sql """drop catalog if exists ${catalogName}"""
+ try {
+ sql """
+ create catalog ${catalogName} properties (
+ ${catalogProps}
+ );
+ """
+ sql """switch ${catalogName};"""
+ try {
+ sql """show databases;"""
+ assertTrue(false)
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains(expectedMessage))
+ }
+ } finally {
+ sql """drop catalog if exists ${catalogName}"""
+ }
+ }
+
+ createCatalogAndQueryTables(
+ catalog1Props,
+ "iceberg_s3tables_catalog_1",
+ partitionDbTableName,
+ nonPartitionDbTableName,
+ expectCounts)
+ createCatalogAndQueryTables(
+ catalog2Props,
+ "iceberg_s3tables_catalog_2",
+ partitionDbTableName,
+ nonPartitionDbTableName,
+ expectCounts)
+ createCatalogAndShowDatabasesException(
+ catalog3Props,
+ "iceberg_s3tables_catalog_3",
+ expectedWebIdentityError)
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
