This is an automated email from the ASF dual-hosted git repository.
takidau pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git
The following commit(s) were added to refs/heads/main by this push:
new cfe7841 Fix register_table to properly initialize FileIO and refactor
overall FileIO initialization to better reveal bugs (#208)
cfe7841 is described below
commit cfe7841fed866071f932b9d1285cb680fc3e77e9
Author: Dennis Huo <[email protected]>
AuthorDate: Mon Aug 26 10:29:07 2024 -0700
Fix register_table to properly initialize FileIO and refactor overall
FileIO initialization to better reveal bugs (#208)
* Instead of injecting a mock STS for integration tests that seek to skip
the credential-subscoping flow, add a feature configuration
"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION" to skip the flow. This also
allows running the server in a mode where "application default" storage
credentials are directly used for file access.
* Refactor the callsites of refreshIOWithCredentials and refreshCredentials
for
consistency and eliminate most of the dependency on the catalogFileIO; in
all non-test
scenarios, it's a bug to actually depend on the catalogFileIO at all, since
all storage
actions need to be done in the context of a Table or View, and such FileIOs
need to
be initialized with properly scoped credentials for those entities.
Make the FileIO refresh correctly inherit table properties. For cases where
we don't
have a concrete table metadata available, use defaultTableProperties
derived from
the standard convention of catalog properties that have the
TABLE_DEFAULT_PREFIX.
* No longer instantiate any catalogFileIO by default; provide a config
switch
so that it's only populated in targeted test scenarios where it's difficult
to restructure callsites to better separate the responsibilities of the
"client-side" vs "server-side"; for now only BasePolarisCatalogTest relies
on the default catalogFileIO. This change also reveals the broken
registerTable across serveral different test classes.
* Override registerTable to explicitly initialize its own FileIO to read
the existing
metadata file so that it no longer relies on the test-only catalogFileIO
which won't
work in isolated server deployments.
---
.../service/catalog/BasePolarisCatalog.java | 232 +++++++++++++--------
.../service/catalog/BasePolarisCatalogTest.java | 1 +
.../PolarisCatalogHandlerWrapperAuthzTest.java | 10 +-
.../catalog/PolarisSparkIntegrationTest.java | 8 +
.../service/test/PolarisConnectionExtension.java | 64 ------
.../resources/polaris-server-integrationtest.yml | 1 +
6 files changed, 165 insertions(+), 151 deletions(-)
diff --git
a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java
b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java
index 8d8d1e9..be0d5e3 100644
---
a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java
+++
b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java
@@ -44,9 +44,11 @@ import java.util.stream.Stream;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.BaseTable;
import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.CatalogUtil;
import org.apache.iceberg.Schema;
+import org.apache.iceberg.Table;
import org.apache.iceberg.TableMetadata;
import org.apache.iceberg.TableMetadataParser;
import org.apache.iceberg.TableOperations;
@@ -66,6 +68,8 @@ import
org.apache.iceberg.exceptions.UnprocessableEntityException;
import org.apache.iceberg.exceptions.ValidationException;
import org.apache.iceberg.io.CloseableGroup;
import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.util.PropertyUtil;
import org.apache.iceberg.view.BaseMetastoreViewCatalog;
import org.apache.iceberg.view.BaseViewOperations;
import org.apache.iceberg.view.ViewBuilder;
@@ -117,6 +121,30 @@ public class BasePolarisCatalog extends
BaseMetastoreViewCatalog
// Config key for whether to allow setting the FILE_IO_IMPL using catalog
properties. Should
// only be allowed in dev/test environments.
static final String ALLOW_SPECIFYING_FILE_IO_IMPL =
"ALLOW_SPECIFYING_FILE_IO_IMPL";
+ static final boolean ALLOW_SPECIFYING_FILE_IO_IMPL_DEFAULT = false;
+
+ // Config key for whether to skip credential-subscoping indirection entirely
whenever trying
+ // to obtain storage credentials for instantiating a FileIO. If 'true', no
attempt is made
+ // to use StorageConfigs to generate table-specific storage credentials, but
instead the default
+ // fallthrough of table-level credential properties or else
provider-specific APPLICATION_DEFAULT
+ // credential-loading will be used for the FileIO.
+ // Typically this setting is used in single-tenant server deployments that
don't rely on
+ // "credential-vending" and can use server-default environment variables or
credential config
+ // files for all storage access, or in test/dev scenarios.
+ static final String SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION =
+ "SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION";
+ static final boolean SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION_DEFAULT = false;
+
+ // Config key for initializing a default "catalogFileIO" that is available
either via getIo()
+ // or for any TableOperations/ViewOperations instantiated, via ops.io()
before entity-specific
+ // FileIO initialization is triggered for any such operations.
+ // Typically this should only be used in test scenarios where a
BasePolarisCatalog instance
+ // is used for both the "client-side" and "server-side" logic instead of
being access through
+ // a REST layer.
+ static final String INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST =
+ "INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST";
+ static final boolean INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST_DEFAULT =
false;
+
private static final int MAX_RETRIES = 12;
static final Predicate<Exception> SHOULD_RETRY_REFRESH_PREDICATE =
@@ -146,6 +174,7 @@ public class BasePolarisCatalog extends
BaseMetastoreViewCatalog
private String defaultBaseLocation;
private CloseableGroup closeableGroup;
private Map<String, String> catalogProperties;
+ private Map<String, String> tableDefaultProperties;
/**
* @param entityManager provides handle to underlying
PolarisMetaStoreManager with which to
@@ -201,11 +230,8 @@ public class BasePolarisCatalog extends
BaseMetastoreViewCatalog
this.defaultBaseLocation = baseLocation.replaceAll("/*$", "");
Boolean allowSpecifyingFileIoImpl =
- callContext
- .getPolarisCallContext()
- .getConfigurationStore()
- .getConfiguration(
- callContext.getPolarisCallContext(),
ALLOW_SPECIFYING_FILE_IO_IMPL, false);
+ getBooleanContextConfiguration(
+ ALLOW_SPECIFYING_FILE_IO_IMPL,
ALLOW_SPECIFYING_FILE_IO_IMPL_DEFAULT);
PolarisStorageConfigurationInfo storageConfigurationInfo =
catalogEntity.getStorageConfigurationInfo();
@@ -228,15 +254,27 @@ public class BasePolarisCatalog extends
BaseMetastoreViewCatalog
ioImplClassName,
storageConfigurationInfo);
}
- this.catalogFileIO = loadFileIO(ioImplClassName, properties);
-
this.closeableGroup = CallContext.getCurrentContext().closeables();
closeableGroup.addCloseable(metricsReporter());
- // TODO: FileIO initialization should should happen later depending on the
operation so
- // we'd also add it to the closeableGroup later.
- closeableGroup.addCloseable(this.catalogFileIO);
closeableGroup.setSuppressCloseFailure(true);
+
catalogProperties = properties;
+ tableDefaultProperties =
+ PropertyUtil.propertiesWithPrefix(properties,
CatalogProperties.TABLE_DEFAULT_PREFIX);
+
+ Boolean initializeDefaultCatalogFileioForTest =
+ getBooleanContextConfiguration(
+ INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST,
+ INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST_DEFAULT);
+ if (Boolean.TRUE.equals(initializeDefaultCatalogFileioForTest)) {
+ LOGGER.debug(
+ "Initializing a default catalogFileIO with properties {}",
tableDefaultProperties);
+ this.catalogFileIO = loadFileIO(ioImplClassName, tableDefaultProperties);
+ closeableGroup.addCloseable(this.catalogFileIO);
+ } else {
+ LOGGER.debug("Not initializing default catalogFileIO");
+ this.catalogFileIO = null;
+ }
}
@Override
@@ -244,6 +282,45 @@ public class BasePolarisCatalog extends
BaseMetastoreViewCatalog
return catalogProperties == null ? ImmutableMap.of() : catalogProperties;
}
+ @Override
+ public Table registerTable(TableIdentifier identifier, String
metadataFileLocation) {
+ Preconditions.checkArgument(
+ identifier != null && isValidIdentifier(identifier), "Invalid
identifier: %s", identifier);
+ Preconditions.checkArgument(
+ metadataFileLocation != null && !metadataFileLocation.isEmpty(),
+ "Cannot register an empty metadata file location as a table");
+
+ // Throw an exception if this table already exists in the catalog.
+ if (tableExists(identifier)) {
+ throw new AlreadyExistsException("Table already exists: %s", identifier);
+ }
+
+ String locationDir = metadataFileLocation.substring(0,
metadataFileLocation.lastIndexOf("/"));
+
+ TableOperations ops = newTableOps(identifier);
+
+ PolarisResolvedPathWrapper resolvedParent =
+ resolvedEntityView.getResolvedPath(identifier.namespace());
+ if (resolvedParent == null) {
+ // Illegal state because the namespace should've already been in the
static resolution set.
+ throw new IllegalStateException(
+ String.format("Failed to fetch resolved parent for TableIdentifier
'%s'", identifier));
+ }
+ FileIO fileIO =
+ refreshIOWithCredentials(
+ identifier,
+ Set.of(locationDir),
+ resolvedParent,
+ new HashMap<>(tableDefaultProperties),
+ Set.of(PolarisStorageActions.READ));
+
+ InputFile metadataFile = fileIO.newInputFile(metadataFileLocation);
+ TableMetadata metadata = TableMetadataParser.read(fileIO, metadataFile);
+ ops.commit(null, metadata);
+
+ return new BaseTable(ops, fullTableName(name(), identifier),
metricsReporter());
+ }
+
@Override
public TableBuilder buildTable(TableIdentifier identifier, Schema schema) {
return new BasePolarisCatalogTableBuilder(identifier, schema);
@@ -312,25 +389,11 @@ public class BasePolarisCatalog extends
BaseMetastoreViewCatalog
}
Optional<PolarisEntity> storageInfoEntity =
findStorageInfo(tableIdentifier);
- if (purge && lastMetadata != null) {
- Map<String, String> credentialsMap =
- storageInfoEntity
- .map(
- entity ->
- refreshCredentials(
- tableIdentifier,
- Set.of(PolarisStorageActions.READ,
PolarisStorageActions.WRITE),
- getLocationsAllowedToBeAccessed(lastMetadata),
- entity))
- .orElse(Map.of());
- Map<String, String> tableProperties = new
HashMap<>(lastMetadata.properties());
- tableProperties.putAll(credentialsMap);
- if (!tableProperties.isEmpty()) {
- catalogFileIO = loadFileIO(ioImplClassName, tableProperties);
- // ensure the new fileIO is closed when the catalog is closed
- closeableGroup.addCloseable(catalogFileIO);
- }
- }
+
+ // The storageProperties we stash away in the Task should be the superset
of the
+ // internalProperties of the StorageInfoEntity to be able to use its
StorageIntegration
+ // combined with other miscellaneous FileIO-related initialization
properties defined
+ // by the Table.
Map<String, String> storageProperties =
storageInfoEntity
.map(PolarisEntity::getInternalPropertiesAsMap)
@@ -339,13 +402,14 @@ public class BasePolarisCatalog extends
BaseMetastoreViewCatalog
if (lastMetadata == null) {
return Map.<String, String>of();
}
- Map<String, String> clone = new HashMap<>(properties);
+ Map<String, String> clone = new HashMap<>();
+
+ // The user-configurable table properties are the baseline,
but then override
+ // with our restricted properties so that table properties
can't clobber the
+ // more restricted ones.
+ clone.putAll(lastMetadata.properties());
clone.put(CatalogProperties.FILE_IO_IMPL, ioImplClassName);
- try {
- clone.putAll(catalogFileIO.properties());
- } catch (UnsupportedOperationException e) {
- LOGGER.warn("FileIO doesn't implement properties()");
- }
+ clone.putAll(properties);
clone.put(PolarisTaskConstants.STORAGE_LOCATION,
lastMetadata.location());
return clone;
})
@@ -794,6 +858,17 @@ public class BasePolarisCatalog extends
BaseMetastoreViewCatalog
// prior to requested subscoped credentials.
tableLocations.forEach(tl -> validateLocationForTableLike(tableIdentifier,
tl));
+ Boolean skipCredentialSubscopingIndirection =
+ getBooleanContextConfiguration(
+ SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION,
SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION_DEFAULT);
+ if (Boolean.TRUE.equals(skipCredentialSubscopingIndirection)) {
+ LOGGER
+ .atInfo()
+ .addKeyValue("tableIdentifier", tableIdentifier)
+ .log("Skipping generation of subscoped creds for table");
+ return Map.of();
+ }
+
boolean allowList =
storageActions.contains(PolarisStorageActions.LIST)
|| storageActions.contains(PolarisStorageActions.ALL);
@@ -1119,16 +1194,18 @@ public class BasePolarisCatalog extends
BaseMetastoreViewCatalog
SHOULD_RETRY_REFRESH_PREDICATE,
MAX_RETRIES,
metadataLocation -> {
- FileIO fileIO = this.tableFileIO;
String latestLocationDir =
latestLocation.substring(0, latestLocation.lastIndexOf('/'));
- fileIO =
+ // TODO: Once we have the "current" table properties pulled into
the resolvedEntity
+ // then we should use the actual current table properties for IO
refresh here
+ // instead of the general tableDefaultProperties.
+ FileIO fileIO =
refreshIOWithCredentials(
tableIdentifier,
Set.of(latestLocationDir),
resolvedEntities,
- new HashMap<>(),
- fileIO);
+ new HashMap<>(tableDefaultProperties),
+ Set.of(PolarisStorageActions.READ));
return TableMetadataParser.read(fileIO, metadataLocation);
});
}
@@ -1164,7 +1241,7 @@ public class BasePolarisCatalog extends
BaseMetastoreViewCatalog
getLocationsAllowedToBeAccessed(metadata),
resolvedStorageEntity,
new HashMap<>(metadata.properties()),
- tableFileIO);
+ Set.of(PolarisStorageActions.READ, PolarisStorageActions.WRITE));
List<PolarisEntity> resolvedNamespace =
resolvedTableEntities == null
@@ -1303,8 +1380,8 @@ public class BasePolarisCatalog extends
BaseMetastoreViewCatalog
private final String fullViewName;
private FileIO viewFileIO;
- BasePolarisViewOperations(FileIO io, TableIdentifier identifier) {
- this.viewFileIO = io;
+ BasePolarisViewOperations(FileIO defaultFileIO, TableIdentifier
identifier) {
+ this.viewFileIO = defaultFileIO;
this.identifier = identifier;
this.fullViewName = ViewUtil.fullViewName(catalogName, identifier);
}
@@ -1336,34 +1413,21 @@ public class BasePolarisCatalog extends
BaseMetastoreViewCatalog
SHOULD_RETRY_REFRESH_PREDICATE,
MAX_RETRIES,
metadataLocation -> {
- FileIO fileIO = this.viewFileIO;
- boolean closeFileIO = false;
String latestLocationDir =
latestLocation.substring(0, latestLocation.lastIndexOf('/'));
- Optional<PolarisEntity> storageInfoEntity =
- findStorageInfoFromHierarchy(resolvedEntities);
- Map<String, String> credentialsMap =
- storageInfoEntity
- .map(
- storageInfo ->
- refreshCredentials(
- identifier,
- Set.of(PolarisStorageActions.READ),
- latestLocationDir,
- storageInfo))
- .orElse(Map.of());
- if (!credentialsMap.isEmpty()) {
- String ioImpl = fileIO.getClass().getName();
- fileIO = loadFileIO(ioImpl, credentialsMap);
- closeFileIO = true;
- }
- try {
- return
ViewMetadataParser.read(fileIO.newInputFile(metadataLocation));
- } finally {
- if (closeFileIO) {
- fileIO.close();
- }
- }
+
+ // TODO: Once we have the "current" table properties pulled into
the resolvedEntity
+ // then we should use the actual current table properties for IO
refresh here
+ // instead of the general tableDefaultProperties.
+ FileIO fileIO =
+ refreshIOWithCredentials(
+ identifier,
+ Set.of(latestLocationDir),
+ resolvedEntities,
+ new HashMap<>(tableDefaultProperties),
+ Set.of(PolarisStorageActions.READ));
+
+ return
ViewMetadataParser.read(fileIO.newInputFile(metadataLocation));
});
}
}
@@ -1414,7 +1478,7 @@ public class BasePolarisCatalog extends
BaseMetastoreViewCatalog
getLocationsAllowedToBeAccessed(metadata),
resolvedStorageEntity,
tableProperties,
- viewFileIO);
+ Set.of(PolarisStorageActions.READ, PolarisStorageActions.WRITE));
String newLocation = writeNewMetadataIfRequired(metadata);
String oldLocation = base == null ? null : currentMetadataLocation();
@@ -1475,17 +1539,13 @@ public class BasePolarisCatalog extends
BaseMetastoreViewCatalog
Set<String> readLocations,
PolarisResolvedPathWrapper resolvedStorageEntity,
Map<String, String> tableProperties,
- FileIO fileIO) {
+ Set<PolarisStorageActions> storageActions) {
Optional<PolarisEntity> storageInfoEntity =
findStorageInfoFromHierarchy(resolvedStorageEntity);
Map<String, String> credentialsMap =
storageInfoEntity
.map(
storageInfo ->
- refreshCredentials(
- identifier,
- Set.of(PolarisStorageActions.READ,
PolarisStorageActions.WRITE),
- readLocations,
- storageInfo))
+ refreshCredentials(identifier, storageActions,
readLocations, storageInfo))
.orElse(Map.of());
// Update the FileIO before we write the new metadata file
@@ -1493,11 +1553,10 @@ public class BasePolarisCatalog extends
BaseMetastoreViewCatalog
// the credentials should always override table-level properties, since
// storage configuration will be found at whatever entity defines it
tableProperties.putAll(credentialsMap);
- if (!tableProperties.isEmpty()) {
- fileIO = loadFileIO(ioImplClassName, tableProperties);
- // ensure the new fileIO is closed when the catalog is closed
- closeableGroup.addCloseable(fileIO);
- }
+ FileIO fileIO = null;
+ fileIO = loadFileIO(ioImplClassName, tableProperties);
+ // ensure the new fileIO is closed when the catalog is closed
+ closeableGroup.addCloseable(fileIO);
return fileIO;
}
@@ -1777,7 +1836,6 @@ public class BasePolarisCatalog extends
BaseMetastoreViewCatalog
// first validate we can read the metadata file
validateLocationForTableLike(tableIdentifier, newLocation);
- TableOperations tableOperations = newTableOps(tableIdentifier);
String locationDir = newLocation.substring(0,
newLocation.lastIndexOf("/"));
FileIO fileIO =
@@ -1785,8 +1843,8 @@ public class BasePolarisCatalog extends
BaseMetastoreViewCatalog
tableIdentifier,
Set.of(locationDir),
resolvedParent,
- new HashMap<>(),
- tableOperations.io());
+ new HashMap<>(tableDefaultProperties),
+ Set.of(PolarisStorageActions.READ, PolarisStorageActions.WRITE));
TableMetadata tableMetadata = TableMetadataParser.read(fileIO,
newLocation);
// then validate that it points to a valid location for this table
@@ -1879,6 +1937,14 @@ public class BasePolarisCatalog extends
BaseMetastoreViewCatalog
}
}
+ /** Helper to retrieve dynamic context-based configuration that has a
boolean value. */
+ private Boolean getBooleanContextConfiguration(String configKey, boolean
defaultValue) {
+ return callContext
+ .getPolarisCallContext()
+ .getConfigurationStore()
+ .getConfiguration(callContext.getPolarisCallContext(), configKey,
defaultValue);
+ }
+
/**
* Check if the exception is retryable for the storage provider
*
diff --git
a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java
b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java
index 12caecb..1652c4c 100644
---
a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java
+++
b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java
@@ -136,6 +136,7 @@ public class BasePolarisCatalogTest extends
CatalogTests<BasePolarisCatalog> {
metaStoreManager =
managerFactory.getOrCreateMetaStoreManager(realmContext);
Map<String, Object> configMap = new HashMap<>();
configMap.put("ALLOW_SPECIFYING_FILE_IO_IMPL", true);
+ configMap.put("INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST", true);
polarisContext =
new PolarisCallContext(
managerFactory.getOrCreateSessionSupplier(realmContext).get(),
diff --git
a/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapperAuthzTest.java
b/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapperAuthzTest.java
index 4f64554..93e8ba3 100644
---
a/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapperAuthzTest.java
+++
b/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapperAuthzTest.java
@@ -21,9 +21,12 @@ package org.apache.polaris.service.catalog;
import com.google.common.collect.ImmutableMap;
import java.time.Instant;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import org.apache.hadoop.conf.Configuration;
import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.SortOrder;
import org.apache.iceberg.TableMetadata;
@@ -1682,12 +1685,11 @@ public class PolarisCatalogHandlerWrapperAuthzTest
extends PolarisAuthzTestBase
Catalog catalog =
super.createCallContextCatalog(
context, authenticatedPolarisPrincipal, resolvedManifest);
+ String fileIoImpl = "org.apache.iceberg.inmemory.InMemoryFileIO";
catalog.initialize(
- externalCatalog,
- ImmutableMap.of(
- CatalogProperties.FILE_IO_IMPL,
"org.apache.iceberg.inmemory.InMemoryFileIO"));
+ externalCatalog,
ImmutableMap.of(CatalogProperties.FILE_IO_IMPL, fileIoImpl));
- FileIO fileIO = ((BasePolarisCatalog)
catalog).newTableOps(table).io();
+ FileIO fileIO = CatalogUtil.loadFileIO(fileIoImpl, Map.of(), new
Configuration());
TableMetadata tableMetadata =
TableMetadata.buildFromEmpty()
.addSchema(SCHEMA, SCHEMA.highestFieldId())
diff --git
a/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisSparkIntegrationTest.java
b/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisSparkIntegrationTest.java
index b948820..c25f884 100644
---
a/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisSparkIntegrationTest.java
+++
b/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisSparkIntegrationTest.java
@@ -141,6 +141,14 @@ public class PolarisSparkIntegrationTest {
CatalogProperties externalProps = new
CatalogProperties("s3://my-bucket/path/to/data");
externalProps.putAll(
Map.of(
+ "table-default.s3.endpoint",
+ s3Container.getHttpEndpoint(),
+ "table-default.s3.path-style-access",
+ "true",
+ "table-default.s3.access-key-id",
+ "foo",
+ "table-default.s3.secret-access-key",
+ "bar",
"s3.endpoint",
s3Container.getHttpEndpoint(),
"s3.path-style-access",
diff --git
a/polaris-service/src/test/java/org/apache/polaris/service/test/PolarisConnectionExtension.java
b/polaris-service/src/test/java/org/apache/polaris/service/test/PolarisConnectionExtension.java
index 30f0a9a..bc267c7 100644
---
a/polaris-service/src/test/java/org/apache/polaris/service/test/PolarisConnectionExtension.java
+++
b/polaris-service/src/test/java/org/apache/polaris/service/test/PolarisConnectionExtension.java
@@ -27,11 +27,8 @@ import io.dropwizard.testing.junit5.DropwizardAppExtension;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.util.Arrays;
-import java.util.EnumMap;
import java.util.Map;
import java.util.Optional;
-import java.util.Set;
-import org.apache.polaris.core.PolarisDiagnostics;
import org.apache.polaris.core.context.CallContext;
import org.apache.polaris.core.context.RealmContext;
import org.apache.polaris.core.entity.PolarisEntityConstants;
@@ -39,17 +36,10 @@ import org.apache.polaris.core.entity.PolarisEntitySubType;
import org.apache.polaris.core.entity.PolarisEntityType;
import org.apache.polaris.core.entity.PolarisGrantRecord;
import org.apache.polaris.core.entity.PolarisPrincipalSecrets;
-import org.apache.polaris.core.persistence.LocalPolarisMetaStoreManagerFactory;
import org.apache.polaris.core.persistence.MetaStoreManagerFactory;
import org.apache.polaris.core.persistence.PolarisMetaStoreManager;
-import org.apache.polaris.core.storage.PolarisCredentialProperty;
-import org.apache.polaris.core.storage.PolarisStorageActions;
-import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo;
-import org.apache.polaris.core.storage.PolarisStorageIntegration;
-import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider;
import org.apache.polaris.service.auth.TokenUtils;
import org.apache.polaris.service.config.PolarisApplicationConfig;
-import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.junit.jupiter.api.extension.BeforeAllCallback;
import org.junit.jupiter.api.extension.ExtensionContext;
@@ -58,12 +48,7 @@ import org.junit.jupiter.api.extension.ParameterContext;
import org.junit.jupiter.api.extension.ParameterResolutionException;
import org.junit.jupiter.api.extension.ParameterResolver;
import org.junit.platform.commons.util.ReflectionUtils;
-import org.mockito.Mockito;
import org.slf4j.LoggerFactory;
-import software.amazon.awssdk.services.sts.StsClient;
-import software.amazon.awssdk.services.sts.model.AssumeRoleRequest;
-import software.amazon.awssdk.services.sts.model.AssumeRoleResponse;
-import software.amazon.awssdk.services.sts.model.Credentials;
public class PolarisConnectionExtension implements BeforeAllCallback,
ParameterResolver {
@@ -94,55 +79,6 @@ public class PolarisConnectionExtension implements
BeforeAllCallback, ParameterR
(PolarisApplicationConfig) dropwizardAppExtension.getConfiguration();
metaStoreManagerFactory = config.getMetaStoreManagerFactory();
- if (metaStoreManagerFactory instanceof
LocalPolarisMetaStoreManagerFactory msmf) {
- StsClient mockSts = Mockito.mock(StsClient.class);
- Mockito.when(mockSts.assumeRole(Mockito.isA(AssumeRoleRequest.class)))
- .thenReturn(
- AssumeRoleResponse.builder()
- .credentials(
- Credentials.builder()
- .accessKeyId("theaccesskey")
- .secretAccessKey("thesecretkey")
- .sessionToken("thesessiontoken")
- .build())
- .build());
- msmf.setStorageIntegrationProvider(
- new PolarisStorageIntegrationProvider() {
- @Override
- public @Nullable <T extends PolarisStorageConfigurationInfo>
- PolarisStorageIntegration<T> getStorageIntegrationForConfig(
- PolarisStorageConfigurationInfo
polarisStorageConfigurationInfo) {
- return new PolarisStorageIntegration<T>("testIntegration") {
- @Override
- public EnumMap<PolarisCredentialProperty, String>
getSubscopedCreds(
- @NotNull PolarisDiagnostics diagnostics,
- @NotNull T storageConfig,
- boolean allowListOperation,
- @NotNull Set<String> allowedReadLocations,
- @NotNull Set<String> allowedWriteLocations) {
- return new EnumMap<>(PolarisCredentialProperty.class);
- }
-
- @Override
- public
EnumMap<PolarisStorageConfigurationInfo.DescribeProperty, String>
- descPolarisStorageConfiguration(
- @NotNull PolarisStorageConfigurationInfo
storageConfigInfo) {
- return new
EnumMap<>(PolarisStorageConfigurationInfo.DescribeProperty.class);
- }
-
- @Override
- public @NotNull Map<String, Map<PolarisStorageActions,
ValidationResult>>
- validateAccessToLocations(
- @NotNull T storageConfig,
- @NotNull Set<PolarisStorageActions> actions,
- @NotNull Set<String> locations) {
- return Map.of();
- }
- };
- }
- });
- }
-
RealmContext realmContext =
config
.getRealmContextResolver()
diff --git
a/polaris-service/src/test/resources/polaris-server-integrationtest.yml
b/polaris-service/src/test/resources/polaris-server-integrationtest.yml
index ebeaf9c..104e59d 100644
--- a/polaris-service/src/test/resources/polaris-server-integrationtest.yml
+++ b/polaris-service/src/test/resources/polaris-server-integrationtest.yml
@@ -71,6 +71,7 @@ featureConfiguration:
DISABLE_TOKEN_GENERATION_FOR_USER_PRINCIPALS: true
ALLOW_WILDCARD_LOCATION: true
ALLOW_SPECIFYING_FILE_IO_IMPL: true
+ SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION: true
ALLOW_OVERLAPPING_CATALOG_URLS: true
SUPPORTED_CATALOG_STORAGE_TYPES:
- FILE
