This is an automated email from the ASF dual-hosted git repository.
snazy pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git
The following commit(s) were added to refs/heads/main by this push:
new e1494bcc3 Require explicit user-consent to enable HadoopFileIO (#1532)
e1494bcc3 is described below
commit e1494bcc33317c10cc22bbc8f8807a0b83fc0576
Author: Robert Stupp <[email protected]>
AuthorDate: Wed May 21 09:36:32 2025 +0200
Require explicit user-consent to enable HadoopFileIO (#1532)
Using `HadoopFileIO` in Polaris can enable "hidden features" that users are
likely not aware of. This change requires users to manually update the
configuration to be able to use `HadoopFileIO` in way that highlights the
consequences of enabling it.
This PR updates Polaris in multiple ways:
* The default of `SUPPORTED_CATALOG_STORAGE_TYPES` is changed to not
include the `FILE` storage type.
* Respect the `ALLOW_SPECIFYING_FILE_IO_IMPL` configuration on namespaces,
tables and views to prevent setting an `io-impl` value for anything but one of
the configured, supported storage-types.
* Unify validation code in a new class `IcebergPropertiesValidation`.
* Using `FILE` or `HadoopFileIO` now _also_ requires the explicit
configuration `ALLOW_INSECURE_STORAGE_TYPES_ACCEPTING_SECURITY_RISKS=true`.
* Added production readiness checks that trigger when
`ALLOW_INSECURE_STORAGE_TYPES_ACCEPTING_SECURITY_RISKS` is `true` or
`SUPPORTED_CATALOG_STORAGE_TYPES` contains `FILE` (defaults and per-realm).
* The two new readiness checks are considered _severe_. Severe
readiness-errors prevent the server from starting up - unless the user
explicitly configured `polaris.readiness.ignore-security-issues=true`.
Log messages and configuration options explicitly use "clear" phrases
highlighting the consequences.
With these changes it is intentionally extremely difficult to start Polaris
with HadoopFileIO. People who work around all these safety nets must have
realized that what they are doing.
A lot of the test code relies on `FILE`/`HadoopFileIO`, those tests got all
the configurations to let those tests continue to work as they are, bypassing
the added security safeguards.
---------
Co-authored-by: Dmitri Bourlatchkov <[email protected]>
---
plugins/spark/v3.5/regtests/docker-compose.yml | 3 +
.../polaris/core/config/FeatureConfiguration.java | 22 +++-
.../core/config/ProductionReadinessCheck.java | 9 +-
.../src/main/resources/application-it.properties | 2 +
.../src/main/resources/application.properties | 2 +-
.../quarkus/config/ProductionReadinessChecks.java | 125 +++++++++++++++++++--
.../config/QuarkusReadinessConfiguration.java} | 21 +++-
.../quarkus/admin/PolarisAuthzTestBase.java | 19 +++-
.../quarkus/admin/PolarisOverlappingTableTest.java | 20 +++-
.../service/quarkus/catalog/GetConfigTest.java | 11 +-
.../catalog/IcebergCatalogHandlerAuthzTest.java | 14 +--
.../quarkus/catalog/IcebergCatalogTest.java | 8 +-
.../quarkus/catalog/IcebergCatalogViewTest.java | 8 +-
.../catalog/PolarisGenericTableCatalogTest.java | 6 +-
.../service/quarkus/catalog/PolicyCatalogTest.java | 6 +-
.../quarkus/catalog/io/FileIOExceptionsTest.java | 11 +-
.../it/QuarkusApplicationIntegrationTest.java | 21 +++-
.../it/QuarkusPolicyServiceIntegrationTest.java | 22 +++-
.../it/QuarkusRestCatalogIntegrationTest.java | 12 +-
.../QuarkusRestCatalogViewFileIntegrationTest.java | 10 +-
regtests/docker-compose.yml | 3 +
.../service/catalog/iceberg/IcebergCatalog.java | 95 +++++++---------
.../catalog/iceberg/IcebergCatalogAdapter.java | 18 +++
.../validation/IcebergPropertiesValidation.java | 93 +++++++++++++++
.../catalog/validation/StorageTypeFileIO.java | 86 ++++++++++++++
.../service/catalog/io/FileIOFactoryTest.java | 9 +-
26 files changed, 548 insertions(+), 108 deletions(-)
diff --git a/plugins/spark/v3.5/regtests/docker-compose.yml
b/plugins/spark/v3.5/regtests/docker-compose.yml
index e1ea1a898..aa0c259fe 100755
--- a/plugins/spark/v3.5/regtests/docker-compose.yml
+++ b/plugins/spark/v3.5/regtests/docker-compose.yml
@@ -28,6 +28,9 @@ services:
POLARIS_BOOTSTRAP_CREDENTIALS: POLARIS,root,secret
quarkus.log.file.enable: "false"
quarkus.otel.sdk.disabled: "true"
+ polaris.features."ALLOW_INSECURE_STORAGE_TYPES": "true"
+ polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES":
"[\"FILE\",\"S3\",\"GCS\",\"AZURE\"]"
+ polaris.readiness.ignore-severe-issues: "true"
healthcheck:
test: ["CMD", "curl", "http://localhost:8182/q/health";]
interval: 10s
diff --git
a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java
b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java
index f06bfad45..930509a20 100644
---
a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java
+++
b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java
@@ -156,8 +156,7 @@ public class FeatureConfiguration<T> extends
PolarisConfiguration<T> {
List.of(
StorageConfigInfo.StorageTypeEnum.S3.name(),
StorageConfigInfo.StorageTypeEnum.AZURE.name(),
- StorageConfigInfo.StorageTypeEnum.GCS.name(),
- StorageConfigInfo.StorageTypeEnum.FILE.name()))
+ StorageConfigInfo.StorageTypeEnum.GCS.name()))
.buildFeatureConfiguration();
public static final FeatureConfiguration<Boolean> CLEANUP_ON_NAMESPACE_DROP =
@@ -269,4 +268,23 @@ public class FeatureConfiguration<T> extends
PolarisConfiguration<T> {
.description("The max number of times to try committing to an
Iceberg table")
.defaultValue(4)
.buildFeatureConfiguration();
+
+ public static final FeatureConfiguration<Boolean>
ALLOW_SPECIFYING_FILE_IO_IMPL =
+ PolarisConfiguration.<Boolean>builder()
+ .key("ALLOW_SPECIFYING_FILE_IO_IMPL")
+ .description(
+ "Config key for whether to allow setting the FILE_IO_IMPL using
catalog properties. "
+ + "Must only be enabled in dev/test environments, should not
be in production systems.")
+ .defaultValue(false)
+ .buildFeatureConfiguration();
+
+ public static final FeatureConfiguration<Boolean>
ALLOW_INSECURE_STORAGE_TYPES =
+ PolarisConfiguration.<Boolean>builder()
+ .key("ALLOW_INSECURE_STORAGE_TYPES")
+ .description(
+ "Allow usage of FileIO implementations that are considered
insecure. "
+ + "Enabling this setting may expose the service to possibly
severe security risks!"
+ + "This should only be set to 'true' for tests!")
+ .defaultValue(false)
+ .buildFeatureConfiguration();
}
diff --git
a/polaris-core/src/main/java/org/apache/polaris/core/config/ProductionReadinessCheck.java
b/polaris-core/src/main/java/org/apache/polaris/core/config/ProductionReadinessCheck.java
index 61d154949..a01e2a076 100644
---
a/polaris-core/src/main/java/org/apache/polaris/core/config/ProductionReadinessCheck.java
+++
b/polaris-core/src/main/java/org/apache/polaris/core/config/ProductionReadinessCheck.java
@@ -48,7 +48,11 @@ public interface ProductionReadinessCheck {
interface Error {
static Error of(String message, String offendingProperty) {
- return ImmutableError.of(message, offendingProperty);
+ return ImmutableError.of(message, offendingProperty, false);
+ }
+
+ static Error ofSevere(String message, String offendingProperty) {
+ return ImmutableError.of(message, offendingProperty, true);
}
@Value.Parameter(order = 1)
@@ -56,5 +60,8 @@ public interface ProductionReadinessCheck {
@Value.Parameter(order = 2)
String offendingProperty();
+
+ @Value.Parameter(order = 3)
+ boolean severe();
}
}
diff --git a/quarkus/defaults/src/main/resources/application-it.properties
b/quarkus/defaults/src/main/resources/application-it.properties
index b172114ca..dc660f54a 100644
--- a/quarkus/defaults/src/main/resources/application-it.properties
+++ b/quarkus/defaults/src/main/resources/application-it.properties
@@ -35,12 +35,14 @@
polaris.features."ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING"=false
polaris.features."ALLOW_EXTERNAL_METADATA_FILE_LOCATION"=false
polaris.features."ALLOW_OVERLAPPING_CATALOG_URLS"=true
polaris.features."ALLOW_SPECIFYING_FILE_IO_IMPL"=true
+polaris.features."ALLOW_INSECURE_STORAGE_TYPES"=true
polaris.features."ALLOW_WILDCARD_LOCATION"=true
polaris.features."ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING"=true
polaris.features."INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_it"=true
polaris.features."SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION"=true
polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES"=["FILE","S3","GCS","AZURE"]
polaris.features."ENABLE_CATALOG_FEDERATION"=true
+polaris.readiness.ignore-severe-issues=true
polaris.realm-context.realms=POLARIS,OTHER
diff --git a/quarkus/defaults/src/main/resources/application.properties
b/quarkus/defaults/src/main/resources/application.properties
index 64f6f2474..b91579011 100644
--- a/quarkus/defaults/src/main/resources/application.properties
+++ b/quarkus/defaults/src/main/resources/application.properties
@@ -109,7 +109,7 @@ polaris.realm-context.header-name=Polaris-Realm
polaris.realm-context.require-header=false
polaris.features."ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING"=false
-polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES"=["S3","GCS","AZURE","FILE"]
+polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES"=["S3","GCS","AZURE"]
# polaris.features."ENABLE_CATALOG_FEDERATION"=true
polaris.features."SUPPORTED_CATALOG_CONNECTION_TYPES"=["ICEBERG_REST"]
diff --git
a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/ProductionReadinessChecks.java
b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/ProductionReadinessChecks.java
index 2389537f1..2168fca55 100644
---
a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/ProductionReadinessChecks.java
+++
b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/ProductionReadinessChecks.java
@@ -18,6 +18,9 @@
*/
package org.apache.polaris.service.quarkus.config;
+import static java.lang.String.format;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.enterprise.event.Observes;
import jakarta.enterprise.event.Startup;
@@ -27,12 +30,15 @@ import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
+import org.apache.polaris.core.config.FeatureConfiguration;
import org.apache.polaris.core.config.ProductionReadinessCheck;
import org.apache.polaris.core.config.ProductionReadinessCheck.Error;
import org.apache.polaris.core.persistence.MetaStoreManagerFactory;
import
org.apache.polaris.service.auth.AuthenticationRealmConfiguration.TokenBrokerConfiguration.RSAKeyPairConfiguration;
import
org.apache.polaris.service.auth.AuthenticationRealmConfiguration.TokenBrokerConfiguration.SymmetricKeyConfiguration;
import org.apache.polaris.service.auth.AuthenticationType;
+import
org.apache.polaris.service.catalog.validation.IcebergPropertiesValidation;
+import org.apache.polaris.service.config.FeaturesConfiguration;
import org.apache.polaris.service.context.DefaultRealmContextResolver;
import org.apache.polaris.service.context.RealmContextResolver;
import org.apache.polaris.service.context.TestRealmContextResolver;
@@ -44,6 +50,7 @@ import org.eclipse.microprofile.config.Config;
import org.eclipse.microprofile.config.ConfigValue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.slf4j.event.Level;
@ApplicationScoped
public class ProductionReadinessChecks {
@@ -57,26 +64,53 @@ public class ProductionReadinessChecks {
*/
private static final String WARNING_SIGN_UTF_8 = "\u0000\u26A0\uFE0F";
+ private static final String SEVERE_SIGN_UTF_8 = "\u0000\uD83D\uDED1";
+
/** A simple warning sign displayed when the character set is not UTF-8. */
private static final String WARNING_SIGN_PLAIN = "!!!";
+ private static final String SEVERE_SIGN_PLAIN = "***STOP***";
+
public void warnOnFailedChecks(
- @Observes Startup event, Instance<ProductionReadinessCheck> checks) {
+ @Observes Startup event,
+ Instance<ProductionReadinessCheck> checks,
+ QuarkusReadinessConfiguration config) {
List<Error> errors = checks.stream().flatMap(check ->
check.getErrors().stream()).toList();
if (!errors.isEmpty()) {
- String warning =
- Charset.defaultCharset().equals(StandardCharsets.UTF_8)
- ? WARNING_SIGN_UTF_8
- : WARNING_SIGN_PLAIN;
- LOGGER.warn("{} Production readiness checks failed! Check the warnings
below.", warning);
+ var utf8 = Charset.defaultCharset().equals(StandardCharsets.UTF_8);
+ var warning = utf8 ? WARNING_SIGN_UTF_8 : WARNING_SIGN_PLAIN;
+ var severe = utf8 ? SEVERE_SIGN_UTF_8 : SEVERE_SIGN_PLAIN;
+ var hasSevere = errors.stream().anyMatch(Error::severe);
+ LOGGER
+ .makeLoggingEventBuilder(hasSevere ? Level.ERROR : Level.WARN)
+ .log(
+ "{} Production readiness checks failed! Check the warnings
below.",
+ hasSevere ? severe : warning);
errors.forEach(
error ->
- LOGGER.warn(
- "- {} Offending configuration option: '{}'.",
- error.message(),
- error.offendingProperty()));
- LOGGER.warn(
- "Refer to
https://polaris.apache.org/in-dev/unreleased/configuring-polaris-for-production
for more information.");
+ LOGGER
+ .makeLoggingEventBuilder(error.severe() ? Level.ERROR :
Level.WARN)
+ .log(
+ "- {} {} Offending configuration option: '{}'.",
+ error.severe() ? severe : warning,
+ error.message(),
+ error.offendingProperty()));
+ LOGGER
+ .makeLoggingEventBuilder(hasSevere ? Level.ERROR : Level.WARN)
+ .log(
+ "Refer to
https://polaris.apache.org/in-dev/unreleased/configuring-polaris-for-production
for more information.");
+
+ if (hasSevere) {
+ if (!config.ignoreSevereIssues()) {
+ throw new IllegalStateException(
+ "Severe production readiness issues detected, startup aborted!");
+ }
+ LOGGER.warn(
+ "{} severe production readiness issues detected, but user
explicitly requested startup by setting "
+ + "polaris.readiness.ignore-severe-issues=true and accepts the
risk of denial-of-service, "
+ + "data-loss, corruption and others !",
+ severe);
+ }
}
}
@@ -176,4 +210,71 @@ public class ProductionReadinessChecks {
private static String authRealmSegment(String realm) {
return realm.equals(QuarkusAuthenticationConfiguration.DEFAULT_REALM_KEY)
? "" : realm + ".";
}
+
+ @Produces
+ public ProductionReadinessCheck checkInsecureStorageSettings(
+ FeaturesConfiguration featureConfiguration) {
+ var insecure = FeatureConfiguration.ALLOW_INSECURE_STORAGE_TYPES;
+
+ var errors = new ArrayList<Error>();
+ if
(Boolean.parseBoolean(featureConfiguration.defaults().get(insecure.key))) {
+ errors.add(
+ Error.ofSevere(
+ "Must not enable a configuration that exposes known and severe
security risks: "
+ + insecure.description,
+ format("polaris.features.\"%s\"", insecure.key)));
+ }
+
+ featureConfiguration
+ .realmOverrides()
+ .forEach(
+ (realmId, overrides) -> {
+ if
(Boolean.parseBoolean(overrides.overrides().get(insecure.key))) {
+ errors.add(
+ Error.ofSevere(
+ "Must not enable a configuration that exposes known
and severe security risks: "
+ + insecure.description,
+ format(
+
"polaris.features.realm-overrides.\"%s\".overrides.\"%s\"",
+ realmId, insecure.key)));
+ }
+ });
+
+ var storageTypes = FeatureConfiguration.SUPPORTED_CATALOG_STORAGE_TYPES;
+ var mapper = new ObjectMapper();
+ var defaults = featureConfiguration.parseDefaults(mapper);
+ var realmOverrides = featureConfiguration.parseRealmOverrides(mapper);
+ @SuppressWarnings("unchecked")
+ var supported = (List<String>) defaults.getOrDefault(storageTypes.key,
List.of());
+ supported.stream()
+ .filter(n -> !IcebergPropertiesValidation.safeStorageType(n))
+ .forEach(
+ t ->
+ errors.add(
+ Error.ofSevere(
+ format(
+ "The storage type '%s' is considered insecure and
exposes the service to severe security risks!",
+ t),
+ format("polaris.features.\"%s\"", storageTypes.key))));
+ realmOverrides.forEach(
+ (realmId, overrides) -> {
+ @SuppressWarnings("unchecked")
+ var s = (List<String>) overrides.getOrDefault(storageTypes.key,
List.of());
+ s.stream()
+ .filter(n -> !IcebergPropertiesValidation.safeStorageType(n))
+ .forEach(
+ t ->
+ errors.add(
+ Error.ofSevere(
+ format(
+ "The storage type '%s' is considered
insecure and exposes the service to severe security risks!",
+ t),
+ format(
+
"polaris.features.realm-overrides.\"%s\".overrides.\"%s\"",
+ realmId, storageTypes.key))));
+ });
+ return errors.isEmpty()
+ ? ProductionReadinessCheck.OK
+ : ProductionReadinessCheck.of(errors.toArray(new Error[0]));
+ }
}
diff --git
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusPolicyServiceIntegrationTest.java
b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusReadinessConfiguration.java
similarity index 58%
copy from
quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusPolicyServiceIntegrationTest.java
copy to
quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusReadinessConfiguration.java
index a668e92e5..3ff56a7ea 100644
---
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusPolicyServiceIntegrationTest.java
+++
b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusReadinessConfiguration.java
@@ -16,10 +16,21 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.polaris.service.quarkus.it;
+package org.apache.polaris.service.quarkus.config;
-import io.quarkus.test.junit.QuarkusTest;
-import org.apache.polaris.service.it.test.PolarisPolicyServiceIntegrationTest;
+import io.quarkus.runtime.annotations.StaticInitSafe;
+import io.smallrye.config.ConfigMapping;
+import io.smallrye.config.WithDefault;
-@QuarkusTest
-public class QuarkusPolicyServiceIntegrationTest extends
PolarisPolicyServiceIntegrationTest {}
+@StaticInitSafe
+@ConfigMapping(prefix = "polaris.readiness")
+public interface QuarkusReadinessConfiguration {
+
+ /**
+ * Setting this to {@code true} means that Polaris will start up even if
severe security risks
+ * have been detected, accepting the risk of denial-of-service, data-loss,
corruption and other
+ * risks.
+ */
+ @WithDefault("false")
+ boolean ignoreSevereIssues();
+}
diff --git
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java
index 8abb4c041..d3a2b0dba 100644
---
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java
+++
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java
@@ -116,7 +116,13 @@ public abstract class PolarisAuthzTestBase {
return Map.of(
"polaris.features.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"",
"true",
+ "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"",
+ "true",
"polaris.features.\"ALLOW_EXTERNAL_METADATA_FILE_LOCATION\"",
+ "true",
+ "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"",
+ "[\"FILE\",\"S3\"]",
+ "polaris.readiness.ignore-severe-issues",
"true");
}
}
@@ -226,9 +232,16 @@ public abstract class PolarisAuthzTestBase {
Map<String, Object> configMap =
Map.of(
- "ALLOW_SPECIFYING_FILE_IO_IMPL", true,
- "ALLOW_EXTERNAL_METADATA_FILE_LOCATION", true,
- "ENABLE_GENERIC_TABLES", true);
+ "ALLOW_SPECIFYING_FILE_IO_IMPL",
+ true,
+ "ALLOW_INSECURE_STORAGE_TYPES",
+ true,
+ "ALLOW_EXTERNAL_METADATA_FILE_LOCATION",
+ true,
+ "SUPPORTED_CATALOG_STORAGE_TYPES",
+ List.of("FILE", "S3"),
+ "ENABLE_GENERIC_TABLES",
+ true);
polarisContext =
new PolarisCallContext(
managerFactory.getOrCreateSessionSupplier(realmContext).get(),
diff --git
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisOverlappingTableTest.java
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisOverlappingTableTest.java
index f2e929484..8ab23593b 100644
---
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisOverlappingTableTest.java
+++
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisOverlappingTableTest.java
@@ -24,6 +24,7 @@ import static
org.apache.polaris.service.quarkus.admin.PolarisAuthzTestBase.SCHE
import static org.assertj.core.api.Assertions.assertThat;
import jakarta.ws.rs.core.Response;
+import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Stream;
@@ -73,10 +74,25 @@ public class PolarisOverlappingTableTest {
static Stream<Arguments> testTableLocationRestrictions() {
Map<String, Object> laxServices =
- Map.of("ALLOW_UNSTRUCTURED_TABLE_LOCATION", "true",
"ALLOW_TABLE_LOCATION_OVERLAP", "true");
+ Map.of(
+ "ALLOW_UNSTRUCTURED_TABLE_LOCATION",
+ "true",
+ "ALLOW_TABLE_LOCATION_OVERLAP",
+ "true",
+ "ALLOW_INSECURE_STORAGE_TYPES",
+ "true",
+ "SUPPORTED_CATALOG_STORAGE_TYPES",
+ List.of("FILE", "S3"));
Map<String, Object> strictServices =
Map.of(
- "ALLOW_UNSTRUCTURED_TABLE_LOCATION", "false",
"ALLOW_TABLE_LOCATION_OVERLAP", "false");
+ "ALLOW_UNSTRUCTURED_TABLE_LOCATION",
+ "false",
+ "ALLOW_TABLE_LOCATION_OVERLAP",
+ "false",
+ "ALLOW_INSECURE_STORAGE_TYPES",
+ "true",
+ "SUPPORTED_CATALOG_STORAGE_TYPES",
+ List.of("FILE", "S3"));
Map<String, Object> laxCatalog =
Map.of(
ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(),
diff --git
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GetConfigTest.java
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GetConfigTest.java
index 5e77d6aea..c497cee6a 100644
---
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GetConfigTest.java
+++
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GetConfigTest.java
@@ -39,7 +39,16 @@ public class GetConfigTest {
@ValueSource(booleans = {true, false})
public void testGetConfig(boolean enableGenericTable) {
TestServices services =
- TestServices.builder().config(Map.of("ENABLE_GENERIC_TABLES",
enableGenericTable)).build();
+ TestServices.builder()
+ .config(
+ Map.of(
+ "ALLOW_INSECURE_STORAGE_TYPES",
+ true,
+ "SUPPORTED_CATALOG_STORAGE_TYPES",
+ List.of("FILE", "S3"),
+ "ENABLE_GENERIC_TABLES",
+ enableGenericTable))
+ .build();
FileStorageConfigInfo fileStorage =
FileStorageConfigInfo.builder(StorageConfigInfo.StorageTypeEnum.FILE)
diff --git
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java
index 926e9791e..5c488809d 100644
---
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java
+++
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java
@@ -80,21 +80,9 @@ import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
@QuarkusTest
-@TestProfile(IcebergCatalogHandlerAuthzTest.Profile.class)
+@TestProfile(PolarisAuthzTestBase.Profile.class)
public class IcebergCatalogHandlerAuthzTest extends PolarisAuthzTestBase {
- public static class Profile extends PolarisAuthzTestBase.Profile {
-
- @Override
- public Map<String, String> getConfigOverrides() {
- return Map.of(
- "polaris.features.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"",
- "true",
- "polaris.features.\"ALLOW_EXTERNAL_METADATA_FILE_LOCATION\"",
- "true");
- }
- }
-
private IcebergCatalogHandler newWrapper() {
return newWrapper(Set.of());
}
diff --git
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java
index b54d032a8..3bd8f45aa 100644
---
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java
+++
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java
@@ -171,14 +171,18 @@ public abstract class IcebergCatalogTest extends
CatalogTests<IcebergCatalog> {
return Map.of(
"polaris.features.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"",
"true",
+ "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"",
+ "true",
"polaris.features.\"INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST\"",
"true",
"polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"",
- "[\"FILE\"]",
+ "[\"FILE\",\"S3\"]",
"polaris.features.\"LIST_PAGINATION_ENABLED\"",
"true",
"polaris.event-listener.type",
- "test");
+ "test",
+ "polaris.readiness.ignore-severe-issues",
+ "true");
}
}
diff --git
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java
index 8b87b75a7..0ac72a1b5 100644
---
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java
+++
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java
@@ -106,12 +106,16 @@ public class IcebergCatalogViewTest extends
ViewCatalogTests<IcebergCatalog> {
"true",
"polaris.features.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"",
"true",
+ "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"",
+ "true",
"polaris.features.\"INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST\"",
"true",
"polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"",
- "[\"FILE\"]",
+ "[\"FILE\",\"S3\"]",
"polaris.event-listener.type",
- "test");
+ "test",
+ "polaris.readiness.ignore-severe-issues",
+ "true");
}
}
diff --git
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java
index 5f8c2d028..267fbb386 100644
---
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java
+++
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java
@@ -108,10 +108,14 @@ public class PolarisGenericTableCatalogTest {
return Map.of(
"polaris.features.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"",
"true",
+ "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"",
+ "true",
"polaris.features.\"INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST\"",
"true",
"polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"",
- "[\"FILE\"]");
+ "[\"FILE\"]",
+ "polaris.readiness.ignore-severe-issues",
+ "true");
}
}
diff --git
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java
index 308971c7c..e94f70c63 100644
---
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java
+++
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java
@@ -126,7 +126,11 @@ public class PolicyCatalogTest {
"polaris.features.\"INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST\"",
"true",
"polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"",
- "[\"FILE\"]");
+ "[\"FILE\"]",
+ "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"",
+ "true",
+ "polaris.readiness.ignore-severe-issues",
+ "true");
}
}
diff --git
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/io/FileIOExceptionsTest.java
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/io/FileIOExceptionsTest.java
index 4ae234e55..abeafeb65 100644
---
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/io/FileIOExceptionsTest.java
+++
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/io/FileIOExceptionsTest.java
@@ -27,6 +27,7 @@ import com.azure.core.exception.AzureException;
import com.google.cloud.storage.StorageException;
import jakarta.ws.rs.core.Response;
import java.util.List;
+import java.util.Map;
import java.util.Optional;
import java.util.stream.Stream;
import org.apache.iceberg.Schema;
@@ -60,7 +61,15 @@ public class FileIOExceptionsTest {
@BeforeAll
public static void beforeAll() {
- services = TestServices.builder().build();
+ services =
+ TestServices.builder()
+ .config(
+ Map.of(
+ "ALLOW_INSECURE_STORAGE_TYPES",
+ true,
+ "SUPPORTED_CATALOG_STORAGE_TYPES",
+ List.of("FILE", "S3")))
+ .build();
ioFactory = (MeasuredFileIOFactory) services.fileIOFactory();
FileStorageConfigInfo storageConfigInfo =
diff --git
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusApplicationIntegrationTest.java
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusApplicationIntegrationTest.java
index 594efc221..56320c905 100644
---
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusApplicationIntegrationTest.java
+++
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusApplicationIntegrationTest.java
@@ -47,14 +47,25 @@ import org.junit.jupiter.api.Test;
public class QuarkusApplicationIntegrationTest extends
PolarisApplicationIntegrationTest {
public static class Profile implements QuarkusTestProfile {
-
@Override
public Map<String, String> getConfigOverrides() {
return Map.of(
- "quarkus.http.limits.max-body-size", "1000000",
- "polaris.realm-context.realms", "POLARIS,OTHER",
- "polaris.features.\"ALLOW_OVERLAPPING_CATALOG_URLS\"", "true",
- "polaris.features.\"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION\"",
"true");
+ "quarkus.http.limits.max-body-size",
+ "1000000",
+ "polaris.realm-context.realms",
+ "POLARIS,OTHER",
+ "polaris.features.\"ALLOW_OVERLAPPING_CATALOG_URLS\"",
+ "true",
+ "polaris.features.\"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION\"",
+ "true",
+ "polaris.features.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"",
+ "true",
+ "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"",
+ "true",
+ "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"",
+ "[\"FILE\",\"S3\"]",
+ "polaris.readiness.ignore-severe-issues",
+ "true");
}
}
diff --git
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusPolicyServiceIntegrationTest.java
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusPolicyServiceIntegrationTest.java
index a668e92e5..a91f26fc0 100644
---
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusPolicyServiceIntegrationTest.java
+++
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusPolicyServiceIntegrationTest.java
@@ -19,7 +19,27 @@
package org.apache.polaris.service.quarkus.it;
import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.QuarkusTestProfile;
+import io.quarkus.test.junit.TestProfile;
+import java.util.Map;
import org.apache.polaris.service.it.test.PolarisPolicyServiceIntegrationTest;
@QuarkusTest
-public class QuarkusPolicyServiceIntegrationTest extends
PolarisPolicyServiceIntegrationTest {}
+@TestProfile(QuarkusPolicyServiceIntegrationTest.Profile.class)
+public class QuarkusPolicyServiceIntegrationTest extends
PolarisPolicyServiceIntegrationTest {
+
+ public static class Profile implements QuarkusTestProfile {
+ @Override
+ public Map<String, String> getConfigOverrides() {
+ return Map.of(
+ "polaris.features.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"",
+ "true",
+ "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"",
+ "true",
+ "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"",
+ "[\"FILE\",\"S3\"]",
+ "polaris.readiness.ignore-severe-issues",
+ "true");
+ }
+ }
+}
diff --git
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogIntegrationTest.java
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogIntegrationTest.java
index 27e29fd61..b2ddd79b0 100644
---
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogIntegrationTest.java
+++
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogIntegrationTest.java
@@ -32,7 +32,17 @@ public class QuarkusRestCatalogIntegrationTest extends
PolarisRestCatalogIntegra
@Override
public Map<String, String> getConfigOverrides() {
- return
Map.of("polaris.features.\"ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING\"",
"false");
+ return Map.of(
+ "polaris.features.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"",
+ "true",
+ "polaris.features.\"ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING\"",
+ "false",
+ "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"",
+ "true",
+ "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"",
+ "[\"FILE\",\"S3\"]",
+ "polaris.readiness.ignore-severe-issues",
+ "true");
}
}
}
diff --git
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewFileIntegrationTest.java
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewFileIntegrationTest.java
index 3987e94d2..0212f86dc 100644
---
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewFileIntegrationTest.java
+++
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewFileIntegrationTest.java
@@ -20,6 +20,7 @@ package org.apache.polaris.service.quarkus.it;
import io.quarkus.test.junit.QuarkusTest;
import io.quarkus.test.junit.QuarkusTestProfile;
+import io.quarkus.test.junit.TestProfile;
import java.lang.reflect.Field;
import java.nio.file.Path;
import java.util.Map;
@@ -29,6 +30,7 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.io.TempDir;
@QuarkusTest
+@TestProfile(QuarkusRestCatalogViewFileIntegrationTest.Profile.class)
public class QuarkusRestCatalogViewFileIntegrationTest
extends PolarisRestCatalogViewFileIntegrationTest {
@@ -36,7 +38,13 @@ public class QuarkusRestCatalogViewFileIntegrationTest
@Override
public Map<String, String> getConfigOverrides() {
- return Map.of("polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"",
"[\"FILE\"]");
+ return Map.of(
+ "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"",
+ "true",
+ "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"",
+ "[\"FILE\"]",
+ "polaris.readiness.ignore-severe-issues",
+ "true");
}
}
diff --git a/regtests/docker-compose.yml b/regtests/docker-compose.yml
index 24ae7a362..cdb44050d 100644
--- a/regtests/docker-compose.yml
+++ b/regtests/docker-compose.yml
@@ -34,6 +34,9 @@ services:
POLARIS_BOOTSTRAP_CREDENTIALS: POLARIS,root,secret
quarkus.log.file.enable: "false"
quarkus.otel.sdk.disabled: "true"
+ polaris.features."ALLOW_INSECURE_STORAGE_TYPES": "true"
+ polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES":
"[\"FILE\",\"S3\",\"GCS\",\"AZURE\"]"
+ polaris.readiness.ignore-severe-issues: "true"
volumes:
- ./credentials:/tmp/credentials/
healthcheck:
diff --git
a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java
b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java
index 2647c5bc6..ba9df38c9 100644
---
a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java
+++
b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java
@@ -71,7 +71,6 @@ import org.apache.iceberg.exceptions.NoSuchTableException;
import org.apache.iceberg.exceptions.NoSuchViewException;
import org.apache.iceberg.exceptions.NotFoundException;
import org.apache.iceberg.exceptions.UnprocessableEntityException;
-import org.apache.iceberg.exceptions.ValidationException;
import org.apache.iceberg.io.CloseableGroup;
import org.apache.iceberg.io.FileIO;
import org.apache.iceberg.io.InputFile;
@@ -92,6 +91,7 @@ import org.apache.polaris.core.admin.model.StorageConfigInfo;
import org.apache.polaris.core.catalog.PolarisCatalogHelpers;
import org.apache.polaris.core.config.BehaviorChangeConfiguration;
import org.apache.polaris.core.config.FeatureConfiguration;
+import org.apache.polaris.core.config.PolarisConfiguration;
import org.apache.polaris.core.context.CallContext;
import org.apache.polaris.core.entity.CatalogEntity;
import org.apache.polaris.core.entity.NamespaceEntity;
@@ -125,6 +125,7 @@ import org.apache.polaris.core.storage.StorageLocation;
import org.apache.polaris.service.catalog.SupportsNotifications;
import org.apache.polaris.service.catalog.io.FileIOFactory;
import org.apache.polaris.service.catalog.io.FileIOUtil;
+import
org.apache.polaris.service.catalog.validation.IcebergPropertiesValidation;
import org.apache.polaris.service.events.AfterTableCommitedEvent;
import org.apache.polaris.service.events.AfterTableRefreshedEvent;
import org.apache.polaris.service.events.AfterViewCommitedEvent;
@@ -147,21 +148,6 @@ public class IcebergCatalog extends
BaseMetastoreViewCatalog
private static final Joiner SLASH = Joiner.on("/");
- // Config key for whether to allow setting the FILE_IO_IMPL using catalog
properties. Should
- // only be allowed in dev/test environments.
- static final String ALLOW_SPECIFYING_FILE_IO_IMPL =
"ALLOW_SPECIFYING_FILE_IO_IMPL";
- static final boolean ALLOW_SPECIFYING_FILE_IO_IMPL_DEFAULT = false;
-
- // Config key for initializing a default "catalogFileIO" that is available
either via getIo()
- // or for any TableOperations/ViewOperations instantiated, via ops.io()
before entity-specific
- // FileIO initialization is triggered for any such operations.
- // Typically this should only be used in test scenarios where a
PolarisIcebergCatalog instance
- // is used for both the "client-side" and "server-side" logic instead of
being access through
- // a REST layer.
- static final String INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST =
- "INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST";
- static final boolean INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST_DEFAULT =
false;
-
public static final Predicate<Exception> SHOULD_RETRY_REFRESH_PREDICATE =
ex -> {
// Default arguments from BaseMetastoreTableOperation only stop
retries on
@@ -258,37 +244,35 @@ public class IcebergCatalog extends
BaseMetastoreViewCatalog
properties.getOrDefault(CatalogProperties.WAREHOUSE_LOCATION, "")));
this.defaultBaseLocation = baseLocation.replaceAll("/*$", "");
- Boolean allowSpecifyingFileIoImpl =
- getBooleanContextConfiguration(
- ALLOW_SPECIFYING_FILE_IO_IMPL,
ALLOW_SPECIFYING_FILE_IO_IMPL_DEFAULT);
+ var storageConfigurationInfo = catalogEntity.getStorageConfigurationInfo();
+ ioImplClassName =
+ IcebergPropertiesValidation.determineFileIOClassName(
+ callContext, properties, storageConfigurationInfo);
- PolarisStorageConfigurationInfo storageConfigurationInfo =
- catalogEntity.getStorageConfigurationInfo();
- if (properties.containsKey(CatalogProperties.FILE_IO_IMPL)) {
- ioImplClassName = properties.get(CatalogProperties.FILE_IO_IMPL);
+ if (ioImplClassName == null) {
+ LOGGER.warn(
+ "Cannot resolve property '{}' for null storageConfiguration.",
+ CatalogProperties.FILE_IO_IMPL);
+ }
- if (!Boolean.TRUE.equals(allowSpecifyingFileIoImpl)) {
- throw new ValidationException(
- "Cannot set property '%s' to '%s' for this catalog.",
- CatalogProperties.FILE_IO_IMPL, ioImplClassName);
- }
+ callContext.closeables().addCloseable(this);
+ this.closeableGroup = new CloseableGroup();
+ closeableGroup.addCloseable(metricsReporter());
+ closeableGroup.setSuppressCloseFailure(true);
+
+ tableDefaultProperties =
+ PropertyUtil.propertiesWithPrefix(properties,
CatalogProperties.TABLE_DEFAULT_PREFIX);
+
+ if (initializeDefaultCatalogFileioForTest()) {
LOGGER.debug(
- "Allowing overriding ioImplClassName to {} for storageConfiguration
{}",
- ioImplClassName,
- storageConfigurationInfo);
+ "Initializing a default catalogFileIO with properties {}",
tableDefaultProperties);
+ this.catalogFileIO = loadFileIO(ioImplClassName, tableDefaultProperties);
+ closeableGroup.addCloseable(this.catalogFileIO);
} else {
- if (storageConfigurationInfo != null) {
- ioImplClassName = storageConfigurationInfo.getFileIoImplClassName();
- LOGGER.debug(
- "Resolved ioImplClassName {} from storageConfiguration {}",
- ioImplClassName,
- storageConfigurationInfo);
- } else {
- LOGGER.warn(
- "Cannot resolve property '{}' for null storageConfiguration.",
- CatalogProperties.FILE_IO_IMPL);
- }
+ LOGGER.debug("Not initializing default catalogFileIO");
+ this.catalogFileIO = null;
}
+
callContext.closeables().addCloseable(this);
this.closeableGroup = new CloseableGroup();
closeableGroup.addCloseable(metricsReporter());
@@ -297,11 +281,7 @@ public class IcebergCatalog extends
BaseMetastoreViewCatalog
tableDefaultProperties =
PropertyUtil.propertiesWithPrefix(properties,
CatalogProperties.TABLE_DEFAULT_PREFIX);
- Boolean initializeDefaultCatalogFileioForTest =
- getBooleanContextConfiguration(
- INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST,
- INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST_DEFAULT);
- if (Boolean.TRUE.equals(initializeDefaultCatalogFileioForTest)) {
+ if (initializeDefaultCatalogFileioForTest()) {
LOGGER.debug(
"Initializing a default catalogFileIO with properties {}",
tableDefaultProperties);
this.catalogFileIO = loadFileIO(ioImplClassName, tableDefaultProperties);
@@ -2565,12 +2545,16 @@ public class IcebergCatalog extends
BaseMetastoreViewCatalog
.getConfiguration(callContext.getPolarisCallContext(), configKey,
defaultValue);
}
+ private boolean initializeDefaultCatalogFileioForTest() {
+ var ctx = callContext.getPolarisCallContext();
+ return ctx.getConfigurationStore()
+ .getConfiguration(ctx, INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST);
+ }
+
private int getMaxMetadataRefreshRetries() {
- return callContext
- .getPolarisCallContext()
- .getConfigurationStore()
- .getConfiguration(
- callContext.getPolarisCallContext(),
FeatureConfiguration.MAX_METADATA_REFRESH_RETRIES);
+ var ctx = callContext.getPolarisCallContext();
+ return ctx.getConfigurationStore()
+ .getConfiguration(ctx,
FeatureConfiguration.MAX_METADATA_REFRESH_RETRIES);
}
/** Build a {@link PageToken} from a string and page size. */
@@ -2590,4 +2574,11 @@ public class IcebergCatalog extends
BaseMetastoreViewCatalog
return PageToken.build(tokenString, pageSize);
}
}
+
+ static final FeatureConfiguration<Boolean>
INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST =
+ PolarisConfiguration.<Boolean>builder()
+ .key("INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST")
+ .defaultValue(false)
+ .description("")
+ .buildFeatureConfiguration();
}
diff --git
a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java
b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java
index 4c9c527d6..c28fd491a 100644
---
a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java
+++
b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java
@@ -19,6 +19,7 @@
package org.apache.polaris.service.catalog.iceberg;
import static
org.apache.polaris.service.catalog.AccessDelegationMode.VENDED_CREDENTIALS;
+import static
org.apache.polaris.service.catalog.validation.IcebergPropertiesValidation.validateIcebergProperties;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
@@ -35,6 +36,7 @@ import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
+import org.apache.iceberg.MetadataUpdate;
import org.apache.iceberg.catalog.Namespace;
import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.exceptions.BadRequestException;
@@ -210,6 +212,7 @@ public class IcebergCatalogAdapter
CreateNamespaceRequest createNamespaceRequest,
RealmContext realmContext,
SecurityContext securityContext) {
+ validateIcebergProperties(callContext,
createNamespaceRequest.properties());
return withCatalog(
securityContext,
prefix,
@@ -301,6 +304,7 @@ public class IcebergCatalogAdapter
UpdateNamespacePropertiesRequest updateNamespacePropertiesRequest,
RealmContext realmContext,
SecurityContext securityContext) {
+ validateIcebergProperties(callContext,
updateNamespacePropertiesRequest.updates());
Namespace ns = decodeNamespace(namespace);
UpdateNamespacePropertiesRequest revisedRequest =
UpdateNamespacePropertiesRequest.builder()
@@ -335,6 +339,7 @@ public class IcebergCatalogAdapter
String accessDelegationMode,
RealmContext realmContext,
SecurityContext securityContext) {
+ validateIcebergProperties(callContext, createTableRequest.properties());
EnumSet<AccessDelegationMode> delegationModes =
parseAccessDelegationModes(accessDelegationMode);
Namespace ns = decodeNamespace(namespace);
@@ -506,6 +511,11 @@ public class IcebergCatalogAdapter
CommitTableRequest commitTableRequest,
RealmContext realmContext,
SecurityContext securityContext) {
+ commitTableRequest.updates().stream()
+ .filter(MetadataUpdate.SetProperties.class::isInstance)
+ .map(MetadataUpdate.SetProperties.class::cast)
+ .forEach(setProperties -> validateIcebergProperties(callContext,
setProperties.updated()));
+
UpdateTableRequest revisedRequest =
UpdateTableRequest.create(
commitTableRequest.identifier(),
@@ -535,6 +545,8 @@ public class IcebergCatalogAdapter
CreateViewRequest createViewRequest,
RealmContext realmContext,
SecurityContext securityContext) {
+ validateIcebergProperties(callContext, createViewRequest.properties());
+
CreateViewRequest revisedRequest =
ImmutableCreateViewRequest.copyOf(createViewRequest)
.withProperties(
@@ -677,6 +689,12 @@ public class IcebergCatalogAdapter
CommitTransactionRequest commitTransactionRequest,
RealmContext realmContext,
SecurityContext securityContext) {
+ commitTransactionRequest.tableChanges().stream()
+ .flatMap(updateTableRequest -> updateTableRequest.updates().stream())
+ .filter(MetadataUpdate.SetProperties.class::isInstance)
+ .map(MetadataUpdate.SetProperties.class::cast)
+ .forEach(setProperties -> validateIcebergProperties(callContext,
setProperties.updated()));
+
CommitTransactionRequest revisedRequest =
new CommitTransactionRequest(
commitTransactionRequest.tableChanges().stream()
diff --git
a/service/common/src/main/java/org/apache/polaris/service/catalog/validation/IcebergPropertiesValidation.java
b/service/common/src/main/java/org/apache/polaris/service/catalog/validation/IcebergPropertiesValidation.java
new file mode 100644
index 000000000..c6f1a5dd9
--- /dev/null
+++
b/service/common/src/main/java/org/apache/polaris/service/catalog/validation/IcebergPropertiesValidation.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.catalog.validation;
+
+import static
org.apache.polaris.core.config.FeatureConfiguration.ALLOW_INSECURE_STORAGE_TYPES;
+import static
org.apache.polaris.core.config.FeatureConfiguration.ALLOW_SPECIFYING_FILE_IO_IMPL;
+import static
org.apache.polaris.core.config.FeatureConfiguration.SUPPORTED_CATALOG_STORAGE_TYPES;
+
+import jakarta.annotation.Nonnull;
+import jakarta.annotation.Nullable;
+import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.polaris.core.context.CallContext;
+import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class IcebergPropertiesValidation {
+ private static final Logger LOGGER =
LoggerFactory.getLogger(IcebergPropertiesValidation.class);
+
+ public static void validateIcebergProperties(
+ @Nonnull CallContext callContext, @Nonnull Map<String, String>
properties) {
+ determineFileIOClassName(callContext, properties, null);
+ }
+
+ public static String determineFileIOClassName(
+ @Nonnull CallContext callContext,
+ @Nonnull Map<String, String> properties,
+ @Nullable PolarisStorageConfigurationInfo storageConfigurationInfo) {
+ var ctx = callContext.getPolarisCallContext();
+ var configStore = ctx.getConfigurationStore();
+
+ var ioImpl = properties.get(CatalogProperties.FILE_IO_IMPL);
+ if (ioImpl != null) {
+ if (!configStore.getConfiguration(ctx, ALLOW_SPECIFYING_FILE_IO_IMPL)) {
+ throw new ValidationException(
+ "Cannot set property '%s' to '%s' for this catalog.",
+ CatalogProperties.FILE_IO_IMPL, ioImpl);
+ }
+ LOGGER.debug(
+ "Allowing overriding ioImplClassName to {} for storageConfiguration
{}",
+ ioImpl,
+ storageConfigurationInfo);
+ } else if (storageConfigurationInfo != null) {
+ ioImpl = storageConfigurationInfo.getFileIoImplClassName();
+ LOGGER.debug(
+ "Resolved ioImplClassName {} from storageConfiguration {}",
+ ioImpl,
+ storageConfigurationInfo);
+ }
+
+ if (ioImpl != null) {
+ var storageType = StorageTypeFileIO.fromFileIoImplementation(ioImpl);
+ if (storageType.validateAllowedStorageType()
+ && !configStore
+ .getConfiguration(ctx, SUPPORTED_CATALOG_STORAGE_TYPES)
+ .contains(storageType.name())) {
+ throw new ValidationException(
+ "File IO implementation '%s', as storage type '%s' is not
supported",
+ ioImpl, storageType);
+ }
+
+ if (!storageType.safe() && !configStore.getConfiguration(ctx,
ALLOW_INSECURE_STORAGE_TYPES)) {
+ throw new ValidationException(
+ "File IO implementation '%s' (storage type '%s') is considered
insecure and must not be used",
+ ioImpl, storageType);
+ }
+ }
+
+ return ioImpl;
+ }
+
+ public static boolean safeStorageType(String name) {
+ return StorageTypeFileIO.valueOf(name).safe();
+ }
+}
diff --git
a/service/common/src/main/java/org/apache/polaris/service/catalog/validation/StorageTypeFileIO.java
b/service/common/src/main/java/org/apache/polaris/service/catalog/validation/StorageTypeFileIO.java
new file mode 100644
index 000000000..55684ab2d
--- /dev/null
+++
b/service/common/src/main/java/org/apache/polaris/service/catalog/validation/StorageTypeFileIO.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.catalog.validation;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo;
+
+enum StorageTypeFileIO {
+ S3("org.apache.iceberg.aws.s3.S3FileIO", true),
+
+ GCS("org.apache.iceberg.gcp.gcs.GCSFileIO", true),
+
+ AZURE("org.apache.iceberg.azure.adlsv2.ADLSFileIO", true),
+
+ FILE("org.apache.iceberg.hadoop.HadoopFileIO", false),
+
+ // Iceberg tests
+ IN_MEMORY("org.apache.iceberg.inmemory.InMemoryFileIO", false, false),
+ ;
+
+ private final String fileIoImplementation;
+ private final boolean safe;
+ private final boolean validateAllowedStorageType;
+
+ StorageTypeFileIO(String fileIoImplementation, boolean safe) {
+ this(fileIoImplementation, safe, true);
+ }
+
+ StorageTypeFileIO(String fileIoImplementation, boolean safe, boolean
validateAllowedStorageType) {
+ this.fileIoImplementation = fileIoImplementation;
+ this.safe = safe;
+ this.validateAllowedStorageType = validateAllowedStorageType;
+ }
+
+ boolean safe() {
+ return safe;
+ }
+
+ boolean validateAllowedStorageType() {
+ return validateAllowedStorageType;
+ }
+
+ static StorageTypeFileIO fromFileIoImplementation(String
fileIoImplementation) {
+ var type = FILE_TO_TO_STORAGE_TYPE.get(fileIoImplementation);
+ if (type == null) {
+ throw new IllegalArgumentException("Unknown FileIO implementation: " +
fileIoImplementation);
+ }
+ return type;
+ }
+
+ private static final Map<String, StorageTypeFileIO> FILE_TO_TO_STORAGE_TYPE;
+
+ static {
+ var map = new HashMap<String, StorageTypeFileIO>();
+ for (var st : PolarisStorageConfigurationInfo.StorageType.values()) {
+ // Ensure all storage types are included in this enum
+ valueOf(st.name());
+ }
+ for (var value : StorageTypeFileIO.values()) {
+ if (value.validateAllowedStorageType()) {
+ // Ensure that the storage type in this enum has a corresponding value
+ PolarisStorageConfigurationInfo.StorageType.valueOf(value.name());
+ }
+ map.put(value.fileIoImplementation, value);
+ }
+ FILE_TO_TO_STORAGE_TYPE = Collections.unmodifiableMap(map);
+ }
+}
diff --git
a/service/common/src/test/java/org/apache/polaris/service/catalog/io/FileIOFactoryTest.java
b/service/common/src/test/java/org/apache/polaris/service/catalog/io/FileIOFactoryTest.java
index 30afc5c09..3dae85660 100644
---
a/service/common/src/test/java/org/apache/polaris/service/catalog/io/FileIOFactoryTest.java
+++
b/service/common/src/test/java/org/apache/polaris/service/catalog/io/FileIOFactoryTest.java
@@ -121,7 +121,14 @@ public class FileIOFactoryTest {
testServices =
TestServices.builder()
- .config(Map.of("ALLOW_SPECIFYING_FILE_IO_IMPL", true))
+ .config(
+ Map.of(
+ "ALLOW_SPECIFYING_FILE_IO_IMPL",
+ true,
+ "ALLOW_INSECURE_STORAGE_TYPES",
+ true,
+ "SUPPORTED_CATALOG_STORAGE_TYPES",
+ List.of("FILE", "S3")))
.realmContext(realmContext)
.stsClient(stsClient)
.fileIOFactorySupplier(fileIOFactorySupplier)
