This is an automated email from the ASF dual-hosted git repository.
yufei pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git
The following commit(s) were added to refs/heads/main by this push:
new fe5b2e540 Core: Add GCP service account impersonation for credentials.
(#3246)
fe5b2e540 is described below
commit fe5b2e540ce03e49519ff5d3a8a5531c7051a1e0
Author: Talat UYARER <[email protected]>
AuthorDate: Thu Dec 11 10:05:16 2025 -0800
Core: Add GCP service account impersonation for credentials. (#3246)
---
CHANGELOG.md | 1 +
gradle/libs.versions.toml | 1 +
polaris-core/build.gradle.kts | 1 +
.../gcp/GcpCredentialsStorageIntegration.java | 72 +++++++++++++++++++++-
.../gcp/GcpCredentialsStorageIntegrationTest.java | 67 ++++++++++++++++++++
5 files changed, 140 insertions(+), 2 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f755d199b..32eec35cb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -62,6 +62,7 @@ request adding CHANGELOG notes for breaking (!) changes and
possibly other secti
### Changes
+- The `gcpServiceAccount` configuration value now affects Polaris behavior
(enables service account impersonation). This value was previously defined but
unused. This change may affect existing deployments that have populated this
property.
- `client.region` is no longer considered a "credential" property (related to
Iceberg REST Catalog API).
- Relaxed the requirements for S3 storage's ARN to allow Polaris to connect to
more non-AWS S3 storage appliances.
- Added checksum to helm deployment so that it will restart when the configmap
has changed.
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
index dc6a954d0..c0c703383 100644
--- a/gradle/libs.versions.toml
+++ b/gradle/libs.versions.toml
@@ -52,6 +52,7 @@ cel-bom = { module = "org.projectnessie.cel:cel-bom", version
= "0.5.3" }
commons-lang3 = { module = "org.apache.commons:commons-lang3", version =
"3.20.0" }
commons-text = { module = "org.apache.commons:commons-text", version =
"1.15.0" }
errorprone = { module = "com.google.errorprone:error_prone_core", version =
"2.45.0" }
+google-cloud-iamcredentials = { module =
"com.google.cloud:google-cloud-iamcredentials", version = "2.60.0" }
google-cloud-storage-bom = { module =
"com.google.cloud:google-cloud-storage-bom", version = "2.60.0" }
guava = { module = "com.google.guava:guava", version = "33.5.0-jre" }
h2 = { module = "com.h2database:h2", version = "2.4.240" }
diff --git a/polaris-core/build.gradle.kts b/polaris-core/build.gradle.kts
index e9427e680..b8beb85b1 100644
--- a/polaris-core/build.gradle.kts
+++ b/polaris-core/build.gradle.kts
@@ -69,6 +69,7 @@ dependencies {
implementation("org.apache.iceberg:iceberg-gcp")
implementation(platform(libs.google.cloud.storage.bom))
implementation("com.google.cloud:google-cloud-storage")
+ implementation(libs.google.cloud.iamcredentials)
testCompileOnly(project(":polaris-immutables"))
testAnnotationProcessor(project(":polaris-immutables", configuration =
"processor"))
diff --git
a/polaris-core/src/main/java/org/apache/polaris/core/storage/gcp/GcpCredentialsStorageIntegration.java
b/polaris-core/src/main/java/org/apache/polaris/core/storage/gcp/GcpCredentialsStorageIntegration.java
index 5f524d9ae..6964e77c7 100644
---
a/polaris-core/src/main/java/org/apache/polaris/core/storage/gcp/GcpCredentialsStorageIntegration.java
+++
b/polaris-core/src/main/java/org/apache/polaris/core/storage/gcp/GcpCredentialsStorageIntegration.java
@@ -20,16 +20,25 @@ package org.apache.polaris.core.storage.gcp;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.api.gax.core.FixedCredentialsProvider;
import com.google.auth.http.HttpTransportFactory;
import com.google.auth.oauth2.AccessToken;
import com.google.auth.oauth2.CredentialAccessBoundary;
import com.google.auth.oauth2.DownscopedCredentials;
import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.iam.credentials.v1.GenerateAccessTokenRequest;
+import com.google.cloud.iam.credentials.v1.GenerateAccessTokenResponse;
+import com.google.cloud.iam.credentials.v1.IamCredentialsClient;
+import com.google.cloud.iam.credentials.v1.IamCredentialsSettings;
import com.google.common.annotations.VisibleForTesting;
+import com.google.protobuf.Duration;
+import com.google.protobuf.Timestamp;
import jakarta.annotation.Nonnull;
import java.io.IOException;
import java.net.URI;
+import java.time.Instant;
import java.util.ArrayList;
+import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -55,6 +64,9 @@ public class GcpCredentialsStorageIntegration
extends InMemoryStorageIntegration<GcpStorageConfigurationInfo> {
private static final Logger LOGGER =
LoggerFactory.getLogger(GcpCredentialsStorageIntegration.class);
+ public static final String SERVICE_ACCOUNT_PREFIX =
"projects/-/serviceAccounts/";
+ public static final String IMPERSONATION_SCOPE =
+ "https://www.googleapis.com/auth/devstorage.read_write";;
private final GoogleCredentials sourceCredentials;
private final HttpTransportFactory transportFactory;
@@ -84,18 +96,20 @@ public class GcpCredentialsStorageIntegration
throw new RuntimeException("Unable to refresh GCP credentials", e);
}
+ GoogleCredentials credentialsToDownscope = getBaseCredentials();
+
CredentialAccessBoundary accessBoundary =
generateAccessBoundaryRules(
allowListOperation, allowedReadLocations, allowedWriteLocations);
DownscopedCredentials credentials =
DownscopedCredentials.newBuilder()
.setHttpTransportFactory(transportFactory)
- .setSourceCredential(sourceCredentials)
+ .setSourceCredential(credentialsToDownscope)
.setCredentialAccessBoundary(accessBoundary)
.build();
AccessToken token;
try {
- token = credentials.refreshAccessToken();
+ token = refreshAccessToken(credentials);
} catch (IOException e) {
LOGGER
.atError()
@@ -123,6 +137,46 @@ public class GcpCredentialsStorageIntegration
return accessConfig.build();
}
+ /**
+ * Returns the credential to be used as the source for downscoping. If a
specific service account
+ * is configured, it impersonates that account first.
+ */
+ private GoogleCredentials getBaseCredentials() {
+ if (config().getGcpServiceAccount() != null) {
+ return createImpersonatedCredentials(sourceCredentials,
config().getGcpServiceAccount());
+ }
+ return sourceCredentials;
+ }
+
+ private GoogleCredentials createImpersonatedCredentials(
+ GoogleCredentials source, String targetServiceAccount) {
+ try (IamCredentialsClient iamCredentialsClient =
createIamCredentialsClient(source)) {
+ GenerateAccessTokenRequest request =
+ GenerateAccessTokenRequest.newBuilder()
+ .setName(SERVICE_ACCOUNT_PREFIX + targetServiceAccount)
+ .addAllDelegates(new ArrayList<>())
+ // 'cloud-platform' is often preferred for impersonation,
+ // but devstorage.read_write is sufficient for GCS specific
operations.
+ // See https://docs.cloud.google.com/storage/docs/oauth-scopes
+ .addScope(IMPERSONATION_SCOPE)
+ .setLifetime(Duration.newBuilder().setSeconds(3600).build())
+ .build();
+
+ GenerateAccessTokenResponse response =
iamCredentialsClient.generateAccessToken(request);
+
+ Timestamp expirationTime = response.getExpireTime();
+ // Use Instant to avoid precision loss or overflow issues with Date
multiplication
+ Date expirationDate =
+ Date.from(Instant.ofEpochSecond(expirationTime.getSeconds(),
expirationTime.getNanos()));
+
+ AccessToken accessToken = new AccessToken(response.getAccessToken(),
expirationDate);
+ return GoogleCredentials.create(accessToken);
+ } catch (IOException e) {
+ throw new RuntimeException(
+ "Unable to impersonate GCP service account: " +
targetServiceAccount, e);
+ }
+ }
+
private String convertToString(CredentialAccessBoundary accessBoundary) {
try {
return new ObjectMapper().writeValueAsString(accessBoundary);
@@ -211,6 +265,20 @@ public class GcpCredentialsStorageIntegration
return accessBoundaryBuilder.build();
}
+ @VisibleForTesting
+ protected AccessToken refreshAccessToken(DownscopedCredentials credentials)
throws IOException {
+ return credentials.refreshAccessToken();
+ }
+
+ @VisibleForTesting
+ protected IamCredentialsClient createIamCredentialsClient(GoogleCredentials
credentials)
+ throws IOException {
+ return IamCredentialsClient.create(
+ IamCredentialsSettings.newBuilder()
+
.setCredentialsProvider(FixedCredentialsProvider.create(credentials))
+ .build());
+ }
+
private static String bucketResource(String bucket) {
return "//storage.googleapis.com/projects/_/buckets/" + bucket;
}
diff --git
a/polaris-core/src/test/java/org/apache/polaris/service/storage/gcp/GcpCredentialsStorageIntegrationTest.java
b/polaris-core/src/test/java/org/apache/polaris/service/storage/gcp/GcpCredentialsStorageIntegrationTest.java
index b0be0883d..74ec73303 100644
---
a/polaris-core/src/test/java/org/apache/polaris/service/storage/gcp/GcpCredentialsStorageIntegrationTest.java
+++
b/polaris-core/src/test/java/org/apache/polaris/service/storage/gcp/GcpCredentialsStorageIntegrationTest.java
@@ -29,13 +29,18 @@ import com.google.api.client.http.javanet.NetHttpTransport;
import com.google.auth.http.HttpTransportFactory;
import com.google.auth.oauth2.AccessToken;
import com.google.auth.oauth2.CredentialAccessBoundary;
+import com.google.auth.oauth2.DownscopedCredentials;
import com.google.auth.oauth2.GoogleCredentials;
import com.google.cloud.ServiceOptions;
+import com.google.cloud.iam.credentials.v1.GenerateAccessTokenRequest;
+import com.google.cloud.iam.credentials.v1.GenerateAccessTokenResponse;
+import com.google.cloud.iam.credentials.v1.IamCredentialsClient;
import com.google.cloud.storage.BlobId;
import com.google.cloud.storage.BlobInfo;
import com.google.cloud.storage.Storage;
import com.google.cloud.storage.StorageException;
import com.google.cloud.storage.StorageOptions;
+import com.google.protobuf.Timestamp;
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
@@ -55,6 +60,7 @@ import
org.assertj.core.api.recursive.comparison.RecursiveComparisonConfiguratio
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
+import org.mockito.Mockito;
class GcpCredentialsStorageIntegrationTest extends BaseStorageIntegrationTest {
@@ -309,6 +315,67 @@ class GcpCredentialsStorageIntegrationTest extends
BaseStorageIntegrationTest {
.isEqualTo(REFRESH_ENDPOINT);
}
+ @Test
+ public void testImpersonation() throws IOException {
+ String serviceAccount = "[email protected]";
+ GcpStorageConfigurationInfo config =
+ GcpStorageConfigurationInfo.builder()
+ .addAllAllowedLocations(List.of("gs://bucket/path"))
+ .gcpServiceAccount(serviceAccount)
+ .build();
+
+ IamCredentialsClient mockIamClient =
Mockito.mock(IamCredentialsClient.class);
+ GenerateAccessTokenResponse mockResponse =
+ GenerateAccessTokenResponse.newBuilder()
+ .setAccessToken("impersonated-token")
+ .setExpireTime(
+ Timestamp.newBuilder().setSeconds(System.currentTimeMillis() /
1000 + 3600).build())
+ .build();
+
Mockito.when(mockIamClient.generateAccessToken(Mockito.any(GenerateAccessTokenRequest.class)))
+ .thenReturn(mockResponse);
+
+ GoogleCredentials mockCreds = Mockito.mock(GoogleCredentials.class);
+
Mockito.when(mockCreds.createScoped(Mockito.any(String.class))).thenReturn(mockCreds);
+
+ GcpCredentialsStorageIntegration integration =
+ new GcpCredentialsStorageIntegration(
+ config,
+ mockCreds,
+ ServiceOptions.getFromServiceLoader(
+ HttpTransportFactory.class, NetHttpTransport::new)) {
+ @Override
+ protected IamCredentialsClient
createIamCredentialsClient(GoogleCredentials credentials) {
+ return mockIamClient;
+ }
+
+ @Override
+ protected AccessToken refreshAccessToken(DownscopedCredentials
credentials) {
+ return new AccessToken("downscoped-token", new Date());
+ }
+ };
+
+ integration.getSubscopedCreds(
+ EMPTY_REALM_CONFIG,
+ true,
+ Set.of("gs://bucket/path"),
+ Set.of("gs://bucket/path"),
+ Optional.empty());
+
+ Mockito.verify(mockIamClient)
+ .generateAccessToken(
+ Mockito.argThat(
+ request ->
+ request
+ .getName()
+ .equals(
+
GcpCredentialsStorageIntegration.SERVICE_ACCOUNT_PREFIX
+ + serviceAccount)
+ && request.getScopeCount() > 0
+ && request
+ .getScope(0)
+
.equals(GcpCredentialsStorageIntegration.IMPERSONATION_SCOPE)));
+ }
+
private boolean isNotNull(JsonNode node) {
return node != null && !node.isNull();
}
