This is an automated email from the ASF dual-hosted git repository.
snazy pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git
The following commit(s) were added to refs/heads/main by this push:
new a28abce00 NoSQL: reduce heap pressure when running tests (#3267)
a28abce00 is described below
commit a28abce00db61910f151a7f3f2d816fa52e7e75f
Author: Robert Stupp <[email protected]>
AuthorDate: Mon Jan 5 16:02:09 2026 +0100
NoSQL: reduce heap pressure when running tests (#3267)
Some tests generate a lot of realms, likely one realm per test case. While
the amount of data per realm is not much, it is nontheless nice to remove that
data immediately (for tests).
The maintenance service, which purges data of eligible realms, cannot be
run against the in-memory backend (different JVM).
This change adds a rather "test only" workaround to purge the realm data in
the in-memory backend immediately.
---
.../metastore/NoSqlMetaStoreManagerFactory.java | 20 ++++++++++++++------
.../nosql/realms/store/RealmStoreImpl.java | 20 ++++++++++++++++++--
.../realms/store/TestRealmStoreIntegration.java | 5 ++++-
3 files changed, 36 insertions(+), 9 deletions(-)
diff --git
a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NoSqlMetaStoreManagerFactory.java
b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NoSqlMetaStoreManagerFactory.java
index 749e0f7dc..6fffd8faa 100644
---
a/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NoSqlMetaStoreManagerFactory.java
+++
b/persistence/nosql/persistence/metastore/src/main/java/org/apache/polaris/persistence/nosql/metastore/NoSqlMetaStoreManagerFactory.java
@@ -198,20 +198,28 @@ class NoSqlMetaStoreManagerFactory implements
MetaStoreManagerFactory {
var nextStatus =
switch (existing.status()) {
- case CREATED, LOADING, INITIALIZING, INACTIVE -> PURGING;
+ case CREATED, LOADING, INITIALIZING, INACTIVE, PURGING ->
+ // PURGING is the state telling the maintenance service to
delete the realm data.
+ PURGING;
case ACTIVE -> INACTIVE;
- case PURGING ->
- // TODO this state should really happen during maintenance!!
+ case PURGED ->
+ // The PURGED state is also handled by the maintenance service.
PURGED;
- case PURGED -> PURGED;
};
+ if (nextStatus == existing.status()) {
+ // No status change (PURGING/PURGED), stop here.
+ // Maintenance service will handle the actual deletion of the realm
data.
+ break;
+ }
+
var update =
RealmDefinition.builder().from(existing).status(nextStatus).build();
var updated = realmManagement.update(existing, update);
- if (updated.status() == PURGED) {
- realmManagement.delete(updated);
+ if (updated.status() == PURGING || updated.status() == PURGED) {
+ // Final status (PURGING/PURGED), stop here.
+ // Maintenance service will handle the actual deletion of the realm
data.
break;
}
}
diff --git
a/persistence/nosql/realms/store-nosql/src/main/java/org/apache/polaris/persistence/nosql/realms/store/RealmStoreImpl.java
b/persistence/nosql/realms/store-nosql/src/main/java/org/apache/polaris/persistence/nosql/realms/store/RealmStoreImpl.java
index 534233ca9..17aa298a5 100644
---
a/persistence/nosql/realms/store-nosql/src/main/java/org/apache/polaris/persistence/nosql/realms/store/RealmStoreImpl.java
+++
b/persistence/nosql/realms/store-nosql/src/main/java/org/apache/polaris/persistence/nosql/realms/store/RealmStoreImpl.java
@@ -24,6 +24,7 @@ import static java.lang.String.format;
import static org.apache.polaris.persistence.nosql.api.Realms.SYSTEM_REALM_ID;
import static
org.apache.polaris.persistence.nosql.api.obj.ObjRef.OBJ_REF_SERIALIZER;
import static org.apache.polaris.persistence.nosql.api.obj.ObjRef.objRef;
+import static
org.apache.polaris.persistence.nosql.realms.api.RealmDefinition.RealmStatus.PURGING;
import static
org.apache.polaris.persistence.nosql.realms.store.RealmsStateObj.REALMS_REF_NAME;
import com.google.common.collect.Streams;
@@ -35,12 +36,14 @@ import java.util.ArrayList;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
+import java.util.Set;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.stream.Stream;
import org.apache.polaris.persistence.nosql.api.Persistence;
import org.apache.polaris.persistence.nosql.api.StreamUtil;
import org.apache.polaris.persistence.nosql.api.SystemPersistence;
+import org.apache.polaris.persistence.nosql.api.backend.Backend;
import org.apache.polaris.persistence.nosql.api.commit.Committer;
import org.apache.polaris.persistence.nosql.api.index.IndexContainer;
import org.apache.polaris.persistence.nosql.api.index.IndexKey;
@@ -54,15 +57,17 @@ import
org.apache.polaris.persistence.nosql.realms.spi.RealmStore;
class RealmStoreImpl implements RealmStore {
private final Persistence systemPersistence;
private final Committer<RealmsStateObj, RealmObj> committer;
+ private final Backend backend;
@SuppressWarnings("CdiInjectionPointsInspection")
@Inject
- RealmStoreImpl(@Nonnull @SystemPersistence Persistence systemPersistence) {
+ RealmStoreImpl(@Nonnull @SystemPersistence Persistence systemPersistence,
Backend backend) {
checkArgument(
SYSTEM_REALM_ID.equals(systemPersistence.realmId()),
"Realms management must happen in the %s realm",
SYSTEM_REALM_ID);
+ this.backend = backend;
this.systemPersistence = systemPersistence;
this.committer =
@@ -206,7 +211,18 @@ class RealmStoreImpl implements RealmStore {
return state.commitResult(obj, newRealms, refObj);
});
- return objToDefinition(realmId, realm.orElseThrow());
+ var result = realm.orElseThrow();
+ if (result.status() == PURGING && "InMemory".equals(backend.type())) {
+ // Handle the test/development case when using the in-memory backend:
+ // In this case there is no chance to run the maintenance service. To
remove no longer
+ // necessary data from the Java heap, call the in-memory backend
directly.
+ // This is "nice behavior" when running tests.
+ // The only "cost" is that the state PURGING may never be pushed to
PURGED, but that is
+ // acceptable.
+ backend.deleteRealms(Set.of(realmId));
+ }
+
+ return objToDefinition(realmId, result);
}
@SuppressWarnings("DuplicatedCode") // looks similar, but extracting isn't
worth it
diff --git
a/persistence/nosql/realms/store-nosql/src/test/java/org/apache/polaris/persistence/nosql/realms/store/TestRealmStoreIntegration.java
b/persistence/nosql/realms/store-nosql/src/test/java/org/apache/polaris/persistence/nosql/realms/store/TestRealmStoreIntegration.java
index 69cf47261..e0d25efc9 100644
---
a/persistence/nosql/realms/store-nosql/src/test/java/org/apache/polaris/persistence/nosql/realms/store/TestRealmStoreIntegration.java
+++
b/persistence/nosql/realms/store-nosql/src/test/java/org/apache/polaris/persistence/nosql/realms/store/TestRealmStoreIntegration.java
@@ -36,6 +36,7 @@ import java.util.function.IntFunction;
import java.util.stream.IntStream;
import org.apache.polaris.persistence.nosql.api.Persistence;
import org.apache.polaris.persistence.nosql.api.PersistenceParams;
+import org.apache.polaris.persistence.nosql.api.backend.Backend;
import
org.apache.polaris.persistence.nosql.realms.api.RealmAlreadyExistsException;
import org.apache.polaris.persistence.nosql.realms.api.RealmDefinition;
import
org.apache.polaris.persistence.nosql.realms.api.RealmExpectedStateMismatchException;
@@ -62,12 +63,14 @@ public class TestRealmStoreIntegration {
@Test
public void nonSystemPersistence() {
+ @SuppressWarnings("resource")
+ var backend = mock(Backend.class);
var nonSystemPersistence = mock(Persistence.class);
var params = mock(PersistenceParams.class);
when(nonSystemPersistence.realmId()).thenReturn("nonSystemPersistence");
when(nonSystemPersistence.params()).thenReturn(params);
soft.assertThatIllegalArgumentException()
- .isThrownBy(() -> new RealmStoreImpl(nonSystemPersistence))
+ .isThrownBy(() -> new RealmStoreImpl(nonSystemPersistence, backend))
.withMessage("Realms management must happen in the ::system:: realm");
}
