This is an automated email from the ASF dual-hosted git repository.
dimas pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git
The following commit(s) were added to refs/heads/main by this push:
new e439ff5b0 feat: Add `endpointInternal` to `AwsStorageConfigInfo`
(#2213)
e439ff5b0 is described below
commit e439ff5b0809220bbc122f321c21423298cf4f2b
Author: Dmitri Bourlatchkov <[email protected]>
AuthorDate: Fri Aug 1 09:15:12 2025 -0400
feat: Add `endpointInternal` to `AwsStorageConfigInfo` (#2213)
* feat: Add `endpointInternal` to `AwsStorageConfigInfo`
This API change is backward compatible with older clients
and server using old storage configuration.
* The `endpointInternal` allows Polaris Servers to use a different
host name (or IP address) for accessing S3 storage than clients.
This is not a common use case, but may be relevant is more complex
environments.
* If not set `endpointInternal` defaults to `endpoint`.
* The STS endpoint default changes to `endpointInternal`.
Contributes to #1530
---
.../apache/polaris/core/entity/CatalogEntity.java | 3 +-
.../aws/AwsCredentialsStorageIntegration.java | 5 ++
.../storage/aws/AwsStorageConfigurationInfo.java | 20 ++++--
.../aws/AwsStorageConfigurationInfoTest.java | 47 ++++++++++++-
.../it/QuarkusRestCatalogMinIOSpecialIT.java | 77 +++++++++++++++-------
spec/polaris-management-service.yml | 15 ++++-
6 files changed, 134 insertions(+), 33 deletions(-)
diff --git
a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java
b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java
index 0de3c3daa..ba8627f06 100644
---
a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java
+++
b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java
@@ -277,7 +277,8 @@ public class CatalogEntity extends PolarisEntity implements
LocationBasedEntity
awsConfigModel.getRegion(),
awsConfigModel.getEndpoint(),
awsConfigModel.getStsEndpoint(),
- awsConfigModel.getPathStyleAccess());
+ awsConfigModel.getPathStyleAccess(),
+ awsConfigModel.getEndpointInternal());
awsConfig.validateArn(awsConfigModel.getRoleArn());
config = awsConfig;
break;
diff --git
a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java
b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java
index f619a9a13..9376d6781 100644
---
a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java
+++
b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java
@@ -120,6 +120,11 @@ public class AwsCredentialsStorageIntegration
if (endpointUri != null) {
accessConfig.put(StorageAccessProperty.AWS_ENDPOINT,
endpointUri.toString());
}
+ URI internalEndpointUri = storageConfig.getInternalEndpointUri();
+ if (internalEndpointUri != null) {
+ accessConfig.putInternalProperty(
+ StorageAccessProperty.AWS_ENDPOINT.getPropertyName(),
internalEndpointUri.toString());
+ }
if (Boolean.TRUE.equals(storageConfig.getPathStyleAccess())) {
accessConfig.put(StorageAccessProperty.AWS_PATH_STYLE_ACCESS,
Boolean.TRUE.toString());
diff --git
a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsStorageConfigurationInfo.java
b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsStorageConfigurationInfo.java
index a007c6a71..bd325d103 100644
---
a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsStorageConfigurationInfo.java
+++
b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsStorageConfigurationInfo.java
@@ -58,6 +58,10 @@ public class AwsStorageConfigurationInfo extends
PolarisStorageConfigurationInfo
@JsonProperty(value = "endpoint")
private @Nullable String endpoint;
+ /** Endpoint URI for internal Polaris calls to S3 API */
+ @JsonProperty(value = "endpointInternal")
+ private @Nullable String endpointInternal;
+
/** Endpoint URI for STS API calls */
@JsonProperty(value = "stsEndpoint")
private @Nullable String stsEndpoint;
@@ -76,7 +80,8 @@ public class AwsStorageConfigurationInfo extends
PolarisStorageConfigurationInfo
@JsonProperty(value = "region", required = false) @Nullable String
region,
@JsonProperty(value = "endpoint") @Nullable String endpoint,
@JsonProperty(value = "stsEndpoint") @Nullable String stsEndpoint,
- @JsonProperty(value = "pathStyleAccess") @Nullable Boolean
pathStyleAccess) {
+ @JsonProperty(value = "pathStyleAccess") @Nullable Boolean
pathStyleAccess,
+ @JsonProperty(value = "endpointInternal") @Nullable String
endpointInternal) {
super(storageType, allowedLocations);
this.roleARN = roleARN;
this.externalId = externalId;
@@ -84,6 +89,7 @@ public class AwsStorageConfigurationInfo extends
PolarisStorageConfigurationInfo
this.endpoint = endpoint;
this.stsEndpoint = stsEndpoint;
this.pathStyleAccess = pathStyleAccess;
+ this.endpointInternal = endpointInternal;
}
public AwsStorageConfigurationInfo(
@@ -91,7 +97,7 @@ public class AwsStorageConfigurationInfo extends
PolarisStorageConfigurationInfo
@Nonnull List<String> allowedLocations,
@Nonnull String roleARN,
@Nullable String region) {
- this(storageType, allowedLocations, roleARN, null, region, null, null,
null);
+ this(storageType, allowedLocations, roleARN, null, region, null, null,
null, null);
}
public AwsStorageConfigurationInfo(
@@ -100,7 +106,7 @@ public class AwsStorageConfigurationInfo extends
PolarisStorageConfigurationInfo
@Nonnull String roleARN,
@Nullable String externalId,
@Nullable String region) {
- this(storageType, allowedLocations, roleARN, externalId, region, null,
null, null);
+ this(storageType, allowedLocations, roleARN, externalId, region, null,
null, null, null);
}
@Override
@@ -160,6 +166,12 @@ public class AwsStorageConfigurationInfo extends
PolarisStorageConfigurationInfo
return endpoint == null ? null : URI.create(endpoint);
}
+ @JsonIgnore
+ @Nullable
+ public URI getInternalEndpointUri() {
+ return endpointInternal == null ? getEndpointUri() :
URI.create(endpointInternal);
+ }
+
/** Returns a flag indicating whether path-style bucket access should be
forced in S3 clients. */
public @Nullable Boolean getPathStyleAccess() {
return pathStyleAccess;
@@ -174,7 +186,7 @@ public class AwsStorageConfigurationInfo extends
PolarisStorageConfigurationInfo
@JsonIgnore
@Nullable
public URI getStsEndpointUri() {
- return stsEndpoint == null ? getEndpointUri() : URI.create(stsEndpoint);
+ return stsEndpoint == null ? getInternalEndpointUri() :
URI.create(stsEndpoint);
}
@JsonIgnore
diff --git
a/polaris-core/src/test/java/org/apache/polaris/core/storage/aws/AwsStorageConfigurationInfoTest.java
b/polaris-core/src/test/java/org/apache/polaris/core/storage/aws/AwsStorageConfigurationInfoTest.java
index f33935b4f..8605b8142 100644
---
a/polaris-core/src/test/java/org/apache/polaris/core/storage/aws/AwsStorageConfigurationInfoTest.java
+++
b/polaris-core/src/test/java/org/apache/polaris/core/storage/aws/AwsStorageConfigurationInfoTest.java
@@ -34,8 +34,13 @@ public class AwsStorageConfigurationInfoTest {
private static AwsStorageConfigurationInfo config(
String endpoint, String stsEndpoint, Boolean pathStyle) {
+ return config(endpoint, stsEndpoint, pathStyle, null);
+ }
+
+ private static AwsStorageConfigurationInfo config(
+ String endpoint, String stsEndpoint, Boolean pathStyle, String
internalEndpoint) {
return new AwsStorageConfigurationInfo(
- S3, List.of(), "role", null, null, endpoint, stsEndpoint, pathStyle);
+ S3, List.of(), "role", null, null, endpoint, stsEndpoint, pathStyle,
internalEndpoint);
}
@Test
@@ -60,6 +65,46 @@ public class AwsStorageConfigurationInfoTest {
AwsStorageConfigurationInfo::getEndpointUri,
AwsStorageConfigurationInfo::getStsEndpointUri)
.containsExactly(URI.create("http://s3.example.com";),
URI.create("http://sts.example.com";));
+ assertThat(config("http://s3.example.com";, null, false,
"http://int.example.com";))
+ .extracting(
+ AwsStorageConfigurationInfo::getEndpointUri,
+ AwsStorageConfigurationInfo::getStsEndpointUri,
+ AwsStorageConfigurationInfo::getInternalEndpointUri)
+ .containsExactly(
+ URI.create("http://s3.example.com";),
+ URI.create("http://int.example.com";),
+ URI.create("http://int.example.com";));
+ }
+
+ @Test
+ public void testInternalEndpoint() {
+ assertThat(config(null, null))
+ .extracting(
+ AwsStorageConfigurationInfo::getEndpointUri,
+ AwsStorageConfigurationInfo::getInternalEndpointUri)
+ .containsExactly(null, null);
+ assertThat(config(null, "http://sts.example.com";))
+ .extracting(
+ AwsStorageConfigurationInfo::getEndpointUri,
+ AwsStorageConfigurationInfo::getInternalEndpointUri)
+ .containsExactly(null, null);
+ assertThat(config("http://s3.example.com";, null))
+ .extracting(
+ AwsStorageConfigurationInfo::getEndpointUri,
+ AwsStorageConfigurationInfo::getInternalEndpointUri)
+ .containsExactly(URI.create("http://s3.example.com";),
URI.create("http://s3.example.com";));
+ assertThat(
+ config(
+ "http://s3.example.com";, "http://sts.example.com";, false,
"http://int.example.com";))
+ .extracting(
+ AwsStorageConfigurationInfo::getEndpointUri,
+ AwsStorageConfigurationInfo::getInternalEndpointUri)
+ .containsExactly(URI.create("http://s3.example.com";),
URI.create("http://int.example.com";));
+ assertThat(config(null, "http://sts.example.com";, false,
"http://int.example.com";))
+ .extracting(
+ AwsStorageConfigurationInfo::getEndpointUri,
+ AwsStorageConfigurationInfo::getInternalEndpointUri)
+ .containsExactly(null, URI.create("http://int.example.com";));
}
@Test
diff --git
a/runtime/service/src/intTest/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogMinIOSpecialIT.java
b/runtime/service/src/intTest/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogMinIOSpecialIT.java
index d75cf2d3a..94f7db17f 100644
---
a/runtime/service/src/intTest/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogMinIOSpecialIT.java
+++
b/runtime/service/src/intTest/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogMinIOSpecialIT.java
@@ -69,6 +69,7 @@ import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.params.ParameterizedTest;
@@ -154,6 +155,14 @@ public class QuarkusRestCatalogMinIOSpecialIT {
private RESTCatalog createCatalog(
Optional<String> endpoint, Optional<String> stsEndpoint, boolean
pathStyleAccess) {
+ return createCatalog(endpoint, stsEndpoint, pathStyleAccess,
Optional.empty());
+ }
+
+ private RESTCatalog createCatalog(
+ Optional<String> endpoint,
+ Optional<String> stsEndpoint,
+ boolean pathStyleAccess,
+ Optional<String> endpointInternal) {
AwsStorageConfigInfo.Builder storageConfig =
AwsStorageConfigInfo.builder()
.setRoleArn("arn:aws:iam::123456789012:role/polaris-test")
@@ -165,6 +174,7 @@ public class QuarkusRestCatalogMinIOSpecialIT {
endpoint.ifPresent(storageConfig::setEndpoint);
stsEndpoint.ifPresent(storageConfig::setStsEndpoint);
+ endpointInternal.ifPresent(storageConfig::setEndpointInternal);
CatalogProperties.Builder catalogProps =
CatalogProperties.builder(storageBase.toASCIIString() + "/" +
catalogName);
@@ -204,39 +214,56 @@ public class QuarkusRestCatalogMinIOSpecialIT {
public void testCreateTable(boolean pathStyle) throws IOException {
try (RESTCatalog restCatalog =
createCatalog(Optional.of(endpoint), Optional.empty(), pathStyle)) {
- catalogApi.createNamespace(catalogName, "test-ns");
- TableIdentifier id = TableIdentifier.of("test-ns", "t1");
- Table table = restCatalog.createTable(id, SCHEMA);
- assertThat(table).isNotNull();
- assertThat(restCatalog.tableExists(id)).isTrue();
-
- TableOperations ops = ((HasTableOperations) table).operations();
- URI location = URI.create(ops.current().metadataFileLocation());
-
- GetObjectResponse response =
- s3Client
- .getObject(
- GetObjectRequest.builder()
- .bucket(location.getAuthority())
- .key(location.getPath().substring(1)) // drop leading
slash
- .build())
- .response();
- assertThat(response.contentLength()).isGreaterThan(0);
-
- LoadTableResponse loadTableResponse =
- catalogApi.loadTableWithAccessDelegation(catalogName, id, "ALL");
- assertThat(loadTableResponse.config()).containsKey("s3.endpoint");
-
+ LoadTableResponse loadTableResponse = doTestCreateTable(restCatalog);
if (pathStyle) {
assertThat(loadTableResponse.config())
.containsEntry("s3.path-style-access", Boolean.TRUE.toString());
}
+ }
+ }
- restCatalog.dropTable(id);
- assertThat(restCatalog.tableExists(id)).isFalse();
+ @Test
+ public void testInternalEndpoints() throws IOException {
+ try (RESTCatalog restCatalog =
+ createCatalog(
+ Optional.of("http://s3.example.com";),
+ Optional.of(endpoint),
+ false,
+ Optional.of(endpoint))) {
+ LoadTableResponse loadTableResponse = doTestCreateTable(restCatalog);
+ assertThat(loadTableResponse.config()).containsEntry("s3.endpoint",
"http://s3.example.com";);
}
}
+ public LoadTableResponse doTestCreateTable(RESTCatalog restCatalog) throws
IOException {
+ catalogApi.createNamespace(catalogName, "test-ns");
+ TableIdentifier id = TableIdentifier.of("test-ns", "t1");
+ Table table = restCatalog.createTable(id, SCHEMA);
+ assertThat(table).isNotNull();
+ assertThat(restCatalog.tableExists(id)).isTrue();
+
+ TableOperations ops = ((HasTableOperations) table).operations();
+ URI location = URI.create(ops.current().metadataFileLocation());
+
+ GetObjectResponse response =
+ s3Client
+ .getObject(
+ GetObjectRequest.builder()
+ .bucket(location.getAuthority())
+ .key(location.getPath().substring(1)) // drop leading slash
+ .build())
+ .response();
+ assertThat(response.contentLength()).isGreaterThan(0);
+
+ LoadTableResponse loadTableResponse =
+ catalogApi.loadTableWithAccessDelegation(catalogName, id, "ALL");
+ assertThat(loadTableResponse.config()).containsKey("s3.endpoint");
+
+ restCatalog.dropTable(id);
+ assertThat(restCatalog.tableExists(id)).isFalse();
+ return loadTableResponse;
+ }
+
@ParameterizedTest
@ValueSource(booleans = {true, false})
public void testAppendFiles(boolean pathStyle) throws IOException {
diff --git a/spec/polaris-management-service.yml
b/spec/polaris-management-service.yml
index 5a2461ebd..a7398bd39 100644
--- a/spec/polaris-management-service.yml
+++ b/spec/polaris-management-service.yml
@@ -1050,12 +1050,23 @@ components:
example: "us-east-2"
endpoint:
type: string
- description: endpoint for S3 requests (optional)
+ description: >-
+ endpoint for S3 requests (optional). Clients always see this
value (if it is set). Polaris Servers
+ may be configured to use a different endpoint URI via the
`endpointInternal` property.
example: "https://s3.example.com:1234";
stsEndpoint:
type: string
- description: endpoint for STS requests (optional). If not set,
defaults to 'endpoint'.
+ description: >-
+ endpoint for STS requests made by the Polaris Server
(optional). If not set, defaults to
+ 'endpointInternal' (which in turn defaults to `endpoint`).
example: "https://sts.example.com:1234";
+ endpointInternal:
+ type: string
+ description: >-
+ endpoint for S3 requests made by the Polaris Server
(optional). If set, Polaris Service will use
+ this value instead of `endpoint`. If not set, defaults to
`endpoint`. Iceberg REST API clients never
+ see this value.
+ example: "https://s3.internal.example.com:1234";
pathStyleAccess:
type: boolean
description: >-
