This is an automated email from the ASF dual-hosted git repository.
yufei pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git
The following commit(s) were added to refs/heads/main by this push:
new 94c94f197 Core: Remove configure:
INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST (#1624)
94c94f197 is described below
commit 94c94f197f1375ac5b7f949f611a4ac03579d9f0
Author: Yufei Gu <[email protected]>
AuthorDate: Wed May 21 11:34:28 2025 -0700
Core: Remove configure: INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST (#1624)
---
.../src/main/resources/application.properties | 1 -
.../quarkus/catalog/IcebergCatalogTest.java | 48 +++++++++++++++++---
.../quarkus/catalog/IcebergCatalogViewTest.java | 2 -
.../catalog/PolarisGenericTableCatalogTest.java | 2 -
.../service/quarkus/catalog/PolicyCatalogTest.java | 2 -
.../service/catalog/iceberg/IcebergCatalog.java | 51 ++++------------------
site/content/in-dev/unreleased/configuration.md | 1 -
7 files changed, 52 insertions(+), 55 deletions(-)
diff --git a/quarkus/defaults/src/main/resources/application.properties
b/quarkus/defaults/src/main/resources/application.properties
index b91579011..a695922d7 100644
--- a/quarkus/defaults/src/main/resources/application.properties
+++ b/quarkus/defaults/src/main/resources/application.properties
@@ -114,7 +114,6 @@
polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES"=["S3","GCS","AZURE"]
polaris.features."SUPPORTED_CATALOG_CONNECTION_TYPES"=["ICEBERG_REST"]
# realm overrides
-#
polaris.features.realm-overrides."my-realm"."INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST"=true
#
polaris.features.realm-overrides."my-realm"."SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION"=true
# polaris.persistence.type=eclipse-link
diff --git
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java
index 638df5baf..7ac194ffd 100644
---
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java
+++
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java
@@ -99,6 +99,7 @@ import
org.apache.polaris.core.persistence.dao.entity.BaseResult;
import org.apache.polaris.core.persistence.dao.entity.EntityResult;
import org.apache.polaris.core.persistence.dao.entity.PrincipalSecretsResult;
import org.apache.polaris.core.persistence.pagination.PageToken;
+import
org.apache.polaris.core.persistence.resolver.PolarisResolutionManifestCatalogView;
import
org.apache.polaris.core.persistence.transactional.TransactionalPersistence;
import org.apache.polaris.core.secrets.UserSecretsManager;
import org.apache.polaris.core.secrets.UserSecretsManagerFactory;
@@ -173,8 +174,6 @@ public abstract class IcebergCatalogTest extends
CatalogTests<IcebergCatalog> {
"true",
"polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"",
"true",
- "polaris.features.\"INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST\"",
- "true",
"polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"",
"[\"FILE\",\"S3\"]",
"polaris.features.\"LIST_PAGINATION_ENABLED\"",
@@ -227,6 +226,45 @@ public abstract class IcebergCatalogTest extends
CatalogTests<IcebergCatalog> {
private TestPolarisEventListener testPolarisEventListener;
private ReservedProperties reservedProperties;
+ /**
+ * A subclass of IcebergCatalog that adds FileIO management capabilities.
This allows the file IO
+ * logic to be encapsulated in a dedicated class.
+ */
+ public static class IcebergFileIOCatalog extends IcebergCatalog {
+
+ public IcebergFileIOCatalog(
+ PolarisEntityManager entityManager,
+ PolarisMetaStoreManager metaStoreManager,
+ CallContext callContext,
+ PolarisResolutionManifestCatalogView resolvedEntityView,
+ SecurityContext securityContext,
+ TaskExecutor taskExecutor,
+ FileIOFactory fileIOFactory,
+ PolarisEventListener polarisEventListener) {
+ super(
+ entityManager,
+ metaStoreManager,
+ callContext,
+ resolvedEntityView,
+ securityContext,
+ taskExecutor,
+ fileIOFactory,
+ polarisEventListener);
+ }
+
+ @Override
+ public synchronized FileIO getIo() {
+ if (catalogFileIO == null) {
+ catalogFileIO = loadFileIO(ioImplClassName, tableDefaultProperties);
+ if (closeableGroup != null) {
+ closeableGroup.addCloseable(catalogFileIO);
+ }
+ }
+
+ return catalogFileIO;
+ }
+ }
+
@BeforeAll
public static void setUpMocks() {
PolarisStorageIntegrationProviderImpl mock =
@@ -372,7 +410,7 @@ public abstract class IcebergCatalogTest extends
CatalogTests<IcebergCatalog> {
callContext, entityManager, securityContext, CATALOG_NAME);
TaskExecutor taskExecutor = Mockito.mock();
IcebergCatalog icebergCatalog =
- new IcebergCatalog(
+ new IcebergFileIOCatalog(
entityManager,
metaStoreManager,
callContext,
@@ -1004,7 +1042,7 @@ public abstract class IcebergCatalogTest extends
CatalogTests<IcebergCatalog> {
callContext, entityManager, securityContext,
catalogWithoutStorage);
TaskExecutor taskExecutor = Mockito.mock();
IcebergCatalog catalog =
- new IcebergCatalog(
+ new IcebergFileIOCatalog(
entityManager,
metaStoreManager,
callContext,
@@ -1071,7 +1109,7 @@ public abstract class IcebergCatalogTest extends
CatalogTests<IcebergCatalog> {
callContext, entityManager, securityContext, catalogName);
TaskExecutor taskExecutor = Mockito.mock();
IcebergCatalog catalog =
- new IcebergCatalog(
+ new IcebergFileIOCatalog(
entityManager,
metaStoreManager,
callContext,
diff --git
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java
index 9cc8a49f2..8b09d243a 100644
---
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java
+++
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java
@@ -108,8 +108,6 @@ public class IcebergCatalogViewTest extends
ViewCatalogTests<IcebergCatalog> {
"true",
"polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"",
"true",
- "polaris.features.\"INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST\"",
- "true",
"polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"",
"[\"FILE\",\"S3\"]",
"polaris.event-listener.type",
diff --git
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java
index 8d5f3c400..3082eda6d 100644
---
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java
+++
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java
@@ -110,8 +110,6 @@ public class PolarisGenericTableCatalogTest {
"true",
"polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"",
"true",
- "polaris.features.\"INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST\"",
- "true",
"polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"",
"[\"FILE\"]",
"polaris.readiness.ignore-severe-issues",
diff --git
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java
index 3bea67411..6e5cebb0d 100644
---
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java
+++
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java
@@ -123,8 +123,6 @@ public class PolicyCatalogTest {
return Map.of(
"polaris.features.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"",
"true",
- "polaris.features.\"INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST\"",
- "true",
"polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"",
"[\"FILE\"]",
"polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"",
diff --git
a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java
b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java
index ba9df38c9..e39a4d0ac 100644
---
a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java
+++
b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java
@@ -91,7 +91,6 @@ import org.apache.polaris.core.admin.model.StorageConfigInfo;
import org.apache.polaris.core.catalog.PolarisCatalogHelpers;
import org.apache.polaris.core.config.BehaviorChangeConfiguration;
import org.apache.polaris.core.config.FeatureConfiguration;
-import org.apache.polaris.core.config.PolarisConfiguration;
import org.apache.polaris.core.context.CallContext;
import org.apache.polaris.core.entity.CatalogEntity;
import org.apache.polaris.core.entity.NamespaceEntity;
@@ -170,14 +169,15 @@ public class IcebergCatalog extends
BaseMetastoreViewCatalog
private final SecurityContext securityContext;
private final PolarisEventListener polarisEventListener;
- private String ioImplClassName;
- private FileIO catalogFileIO;
+ protected String ioImplClassName;
+ protected FileIO catalogFileIO;
+ protected CloseableGroup closeableGroup;
+ protected Map<String, String> tableDefaultProperties;
+
private final String catalogName;
private long catalogId = -1;
private String defaultBaseLocation;
- private CloseableGroup closeableGroup;
private Map<String, String> catalogProperties;
- private Map<String, String> tableDefaultProperties;
private FileIOFactory fileIOFactory;
private PolarisMetaStoreManager metaStoreManager;
@@ -263,16 +263,6 @@ public class IcebergCatalog extends
BaseMetastoreViewCatalog
tableDefaultProperties =
PropertyUtil.propertiesWithPrefix(properties,
CatalogProperties.TABLE_DEFAULT_PREFIX);
- if (initializeDefaultCatalogFileioForTest()) {
- LOGGER.debug(
- "Initializing a default catalogFileIO with properties {}",
tableDefaultProperties);
- this.catalogFileIO = loadFileIO(ioImplClassName, tableDefaultProperties);
- closeableGroup.addCloseable(this.catalogFileIO);
- } else {
- LOGGER.debug("Not initializing default catalogFileIO");
- this.catalogFileIO = null;
- }
-
callContext.closeables().addCloseable(this);
this.closeableGroup = new CloseableGroup();
closeableGroup.addCloseable(metricsReporter());
@@ -280,16 +270,6 @@ public class IcebergCatalog extends
BaseMetastoreViewCatalog
tableDefaultProperties =
PropertyUtil.propertiesWithPrefix(properties,
CatalogProperties.TABLE_DEFAULT_PREFIX);
-
- if (initializeDefaultCatalogFileioForTest()) {
- LOGGER.debug(
- "Initializing a default catalogFileIO with properties {}",
tableDefaultProperties);
- this.catalogFileIO = loadFileIO(ioImplClassName, tableDefaultProperties);
- closeableGroup.addCloseable(this.catalogFileIO);
- } else {
- LOGGER.debug("Not initializing default catalogFileIO");
- this.catalogFileIO = null;
- }
}
public void setMetaStoreManager(PolarisMetaStoreManager newMetaStoreManager)
{
@@ -359,8 +339,7 @@ public class IcebergCatalog extends BaseMetastoreViewCatalog
@VisibleForTesting
public TableOperations newTableOps(
TableIdentifier tableIdentifier, boolean makeMetadataCurrentOnCommit) {
- return new BasePolarisTableOperations(
- catalogFileIO, tableIdentifier, makeMetadataCurrentOnCommit);
+ return new BasePolarisTableOperations(getIo(), tableIdentifier,
makeMetadataCurrentOnCommit);
}
@Override
@@ -862,9 +841,10 @@ public class IcebergCatalog extends
BaseMetastoreViewCatalog
return listTableLike(PolarisEntitySubType.ICEBERG_VIEW, namespace,
pageToken);
}
+ @VisibleForTesting
@Override
protected ViewOperations newViewOps(TableIdentifier identifier) {
- return new BasePolarisViewOperations(catalogFileIO, identifier);
+ return new BasePolarisViewOperations(getIo(), identifier);
}
@Override
@@ -2514,7 +2494,7 @@ public class IcebergCatalog extends
BaseMetastoreViewCatalog
* @param properties used to initialize the FileIO implementation
* @return FileIO object
*/
- private FileIO loadFileIO(String ioImpl, Map<String, String> properties) {
+ protected FileIO loadFileIO(String ioImpl, Map<String, String> properties) {
IcebergTableLikeEntity icebergTableLikeEntity =
IcebergTableLikeEntity.of(catalogEntity);
TableIdentifier identifier = icebergTableLikeEntity.getTableIdentifier();
Set<String> locations = Set.of(catalogEntity.getDefaultBaseLocation());
@@ -2545,12 +2525,6 @@ public class IcebergCatalog extends
BaseMetastoreViewCatalog
.getConfiguration(callContext.getPolarisCallContext(), configKey,
defaultValue);
}
- private boolean initializeDefaultCatalogFileioForTest() {
- var ctx = callContext.getPolarisCallContext();
- return ctx.getConfigurationStore()
- .getConfiguration(ctx, INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST);
- }
-
private int getMaxMetadataRefreshRetries() {
var ctx = callContext.getPolarisCallContext();
return ctx.getConfigurationStore()
@@ -2574,11 +2548,4 @@ public class IcebergCatalog extends
BaseMetastoreViewCatalog
return PageToken.build(tokenString, pageSize);
}
}
-
- static final FeatureConfiguration<Boolean>
INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST =
- PolarisConfiguration.<Boolean>builder()
- .key("INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST")
- .defaultValue(false)
- .description("")
- .buildFeatureConfiguration();
}
diff --git a/site/content/in-dev/unreleased/configuration.md
b/site/content/in-dev/unreleased/configuration.md
index e9802c2b2..b195d05a5 100644
--- a/site/content/in-dev/unreleased/configuration.md
+++ b/site/content/in-dev/unreleased/configuration.md
@@ -90,7 +90,6 @@ read-only mode, as Polaris only reads the configuration file
once, at startup.
| `polaris.realm-context.header-name`
| `Polaris-Realm` | Define the header name defining the realm
context.
|
| `polaris.features."ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING"`
| `false` | Flag to enforce check if credential
rotation.
|
| `polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES"`
| `FILE` | Define the catalog supported storage.
Supported values are `S3`, `GCS`, `AZURE`, `FILE`.
|
-|
`polaris.features.realm-overrides."my-realm"."INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST"`
| `true` | "Override" realm features, here the catalog init
default flag.
|
|
`polaris.features.realm-overrides."my-realm"."SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION"`
| `true` | "Override" realm features, here the skip
credential subscoping indirection flag.
|
| `polaris.authentication.authenticator.type`
| `default` | Define the Polaris authenticator type.
|
| `polaris.authentication.token-service.type`
| `default` | Define the Polaris token service type.
|
