This is an automated email from the ASF dual-hosted git repository.
collado pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git
The following commit(s) were added to refs/heads/main by this push:
new b8446866 pass skip_credential_subscoping_indirection param to
TaskFileIOSupplier (#400)
b8446866 is described below
commit b8446866b55e79c35c6505681c871e22f8a86634
Author: Alessandro Nori <[email protected]>
AuthorDate: Wed Dec 18 18:46:44 2024 +0100
pass skip_credential_subscoping_indirection param to TaskFileIOSupplier
(#400)
* pass skip_credential_subscoping_indirection param to TaskFileIOSupplier
# Conflicts:
#
dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/PolarisApplication.java
* address review comments + reformat
* fix ctor in test
* pass configurationStore to TaskFileIOSupplier
---
.../service/dropwizard/PolarisApplication.java | 6 +++-
.../dropwizard/catalog/BasePolarisCatalogTest.java | 5 ++-
.../apache/polaris/core/PolarisConfiguration.java | 15 +++++++++
.../service/catalog/BasePolarisCatalog.java | 15 ++-------
.../polaris/service/task/TaskFileIOSupplier.java | 37 +++++++++++++++-------
5 files changed, 52 insertions(+), 26 deletions(-)
diff --git
a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/PolarisApplication.java
b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/PolarisApplication.java
index 62f4d304..29afa31a 100644
---
a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/PolarisApplication.java
+++
b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/PolarisApplication.java
@@ -82,6 +82,7 @@ import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.iceberg.rest.RESTSerializers;
+import org.apache.polaris.core.PolarisConfigurationStore;
import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal;
import org.apache.polaris.core.auth.PolarisAuthorizer;
import org.apache.polaris.core.auth.PolarisAuthorizerImpl;
@@ -329,7 +330,10 @@ public class PolarisApplication extends
Application<PolarisApplicationConfig> {
TaskExecutorImpl taskExecutor =
new TaskExecutorImpl(taskConfig.executorService(),
metaStoreManagerFactory);
TaskFileIOSupplier fileIOSupplier =
- new TaskFileIOSupplier(metaStoreManagerFactory,
fileIOFactory);
+ new TaskFileIOSupplier(
+ metaStoreManagerFactory,
+ fileIOFactory,
+
configuration.findService(PolarisConfigurationStore.class));
taskExecutor.addTaskHandler(
new TableCleanupTaskHandler(
taskExecutor, metaStoreManagerFactory,
fileIOSupplier));
diff --git
a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/BasePolarisCatalogTest.java
b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/BasePolarisCatalogTest.java
index 436ce4cc..1b7c59e2 100644
---
a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/BasePolarisCatalogTest.java
+++
b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/BasePolarisCatalogTest.java
@@ -1357,7 +1357,10 @@ public class BasePolarisCatalogTest extends
CatalogTests<BasePolarisCatalog> {
.containsEntry(PolarisCredentialProperty.AWS_SECRET_KEY,
SECRET_ACCESS_KEY)
.containsEntry(PolarisCredentialProperty.AWS_TOKEN, SESSION_TOKEN);
FileIO fileIO =
- new TaskFileIOSupplier(createMockMetaStoreManagerFactory(), new
DefaultFileIOFactory())
+ new TaskFileIOSupplier(
+ createMockMetaStoreManagerFactory(),
+ new DefaultFileIOFactory(),
+ polarisContext.getConfigurationStore())
.apply(taskEntity);
Assertions.assertThat(fileIO).isNotNull().isInstanceOf(InMemoryFileIO.class);
}
diff --git
a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java
b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java
index 45a2f3c2..294a6ac6 100644
---
a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java
+++
b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java
@@ -110,6 +110,21 @@ public class PolarisConfiguration<T> {
.defaultValue(false)
.build();
+ public static final PolarisConfiguration<Boolean>
SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION =
+ PolarisConfiguration.<Boolean>builder()
+ .key("SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION")
+ .description(
+ "If set to true, skip credential-subscoping indirection entirely
whenever trying\n"
+ + " to obtain storage credentials for instantiating a
FileIO. If 'true', no attempt is made\n"
+ + " to use StorageConfigs to generate table-specific
storage credentials, but instead the default\n"
+ + " fallthrough of table-level credential properties or
else provider-specific APPLICATION_DEFAULT\n"
+ + " credential-loading will be used for the FileIO.\n"
+ + " Typically this setting is used in single-tenant server
deployments that don't rely on\n"
+ + " \"credential-vending\" and can use server-default
environment variables or credential config\n"
+ + " files for all storage access, or in test/dev
scenarios.")
+ .defaultValue(false)
+ .build();
+
public static final PolarisConfiguration<Boolean>
ALLOW_TABLE_LOCATION_OVERLAP =
PolarisConfiguration.<Boolean>builder()
.key("ALLOW_TABLE_LOCATION_OVERLAP")
diff --git
a/service/common/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java
b/service/common/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java
index a0f85911..56fc2468 100644
---
a/service/common/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java
+++
b/service/common/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java
@@ -121,18 +121,6 @@ public class BasePolarisCatalog extends
BaseMetastoreViewCatalog
static final String ALLOW_SPECIFYING_FILE_IO_IMPL =
"ALLOW_SPECIFYING_FILE_IO_IMPL";
static final boolean ALLOW_SPECIFYING_FILE_IO_IMPL_DEFAULT = false;
- // Config key for whether to skip credential-subscoping indirection entirely
whenever trying
- // to obtain storage credentials for instantiating a FileIO. If 'true', no
attempt is made
- // to use StorageConfigs to generate table-specific storage credentials, but
instead the default
- // fallthrough of table-level credential properties or else
provider-specific APPLICATION_DEFAULT
- // credential-loading will be used for the FileIO.
- // Typically this setting is used in single-tenant server deployments that
don't rely on
- // "credential-vending" and can use server-default environment variables or
credential config
- // files for all storage access, or in test/dev scenarios.
- static final String SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION =
- "SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION";
- static final boolean SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION_DEFAULT = false;
-
// Config key for initializing a default "catalogFileIO" that is available
either via getIo()
// or for any TableOperations/ViewOperations instantiated, via ops.io()
before entity-specific
// FileIO initialization is triggered for any such operations.
@@ -874,7 +862,8 @@ public class BasePolarisCatalog extends
BaseMetastoreViewCatalog
PolarisEntity entity) {
Boolean skipCredentialSubscopingIndirection =
getBooleanContextConfiguration(
- SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION,
SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION_DEFAULT);
+ PolarisConfiguration.SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION.key,
+
PolarisConfiguration.SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION.defaultValue);
if (Boolean.TRUE.equals(skipCredentialSubscopingIndirection)) {
LOGGER
.atInfo()
diff --git
a/service/common/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java
b/service/common/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java
index c84eebd9..75ceff8e 100644
---
a/service/common/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java
+++
b/service/common/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java
@@ -24,6 +24,8 @@ import java.util.Set;
import java.util.function.Function;
import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.io.FileIO;
+import org.apache.polaris.core.PolarisConfiguration;
+import org.apache.polaris.core.PolarisConfigurationStore;
import org.apache.polaris.core.context.CallContext;
import org.apache.polaris.core.entity.PolarisTaskConstants;
import org.apache.polaris.core.entity.TaskEntity;
@@ -34,11 +36,15 @@ import org.apache.polaris.service.catalog.io.FileIOFactory;
public class TaskFileIOSupplier implements Function<TaskEntity, FileIO> {
private final MetaStoreManagerFactory metaStoreManagerFactory;
private final FileIOFactory fileIOFactory;
+ private final PolarisConfigurationStore configurationStore;
public TaskFileIOSupplier(
- MetaStoreManagerFactory metaStoreManagerFactory, FileIOFactory
fileIOFactory) {
+ MetaStoreManagerFactory metaStoreManagerFactory,
+ FileIOFactory fileIOFactory,
+ PolarisConfigurationStore configurationStore) {
this.metaStoreManagerFactory = metaStoreManagerFactory;
this.fileIOFactory = fileIOFactory;
+ this.configurationStore = configurationStore;
}
@Override
@@ -49,16 +55,25 @@ public class TaskFileIOSupplier implements
Function<TaskEntity, FileIO> {
metaStoreManagerFactory.getOrCreateMetaStoreManager(
CallContext.getCurrentContext().getRealmContext());
Map<String, String> properties = new HashMap<>(internalProperties);
- properties.putAll(
- metaStoreManagerFactory
-
.getOrCreateStorageCredentialCache(CallContext.getCurrentContext().getRealmContext())
- .getOrGenerateSubScopeCreds(
- metaStoreManager,
- CallContext.getCurrentContext().getPolarisCallContext(),
- task,
- true,
- Set.of(location),
- Set.of(location)));
+
+ Boolean skipCredentialSubscopingIndirection =
+ configurationStore.getConfiguration(
+ null,
+ PolarisConfiguration.SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION.key,
+
PolarisConfiguration.SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION.defaultValue);
+
+ if (!skipCredentialSubscopingIndirection) {
+ properties.putAll(
+ metaStoreManagerFactory
+
.getOrCreateStorageCredentialCache(CallContext.getCurrentContext().getRealmContext())
+ .getOrGenerateSubScopeCreds(
+ metaStoreManager,
+ CallContext.getCurrentContext().getPolarisCallContext(),
+ task,
+ true,
+ Set.of(location),
+ Set.of(location)));
+ }
String ioImpl =
properties.getOrDefault(
CatalogProperties.FILE_IO_IMPL,
"org.apache.iceberg.io.ResolvingFileIO");
