This is an automated email from the ASF dual-hosted git repository.
roryqi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new e7c33c2471 [#14298] improvement(tms): Import only missing
schemas/tables on policy expand (#14328)
e7c33c2471 is described below
commit e7c33c247142878d9928db7d276a212a9f87d209
Author: MaSai <[email protected]>
AuthorDate: Thu Oct 29 15:06:53 2026 +0800
[#14298] improvement(tms): Import only missing schemas/tables on policy
expand (#14328)
### What changes were proposed in this pull request?
- Prefer live `listSchemas`/`listTables` when expanding catalog/schema
tags in `NearestPolicyTableEnumerator`.
- Import only missing schema/table entities into the relational store
(instead of always loading everything).
Split out of #14300 so the iceberg catalog-wrapper cache change stays
separate from TMS enumerator work.
### Why are the changes needed?
Policy expand should avoid redundant import/load work when
schemas/tables already exist in entity store, while still discovering
live tables from the catalog.
Fixes #14298 (TMS portion)
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
- `TestNearestPolicyTableEnumerator`
Made with [Cursor](https://cursor.com)
---------
Co-authored-by: Cursor <[email protected]>
---
.../expand/NearestPolicyTableEnumerator.java | 443 ++++++++++++++++-----
.../expand/TestNearestPolicyTableEnumerator.java | 122 +++++-
2 files changed, 471 insertions(+), 94 deletions(-)
diff --git
a/plugins/maintenance/src/main/java/org/apache/gravitino/maintenance/scheduler/policy/expand/NearestPolicyTableEnumerator.java
b/plugins/maintenance/src/main/java/org/apache/gravitino/maintenance/scheduler/policy/expand/NearestPolicyTableEnumerator.java
index 35b5ee2d12..938e54767b 100644
---
a/plugins/maintenance/src/main/java/org/apache/gravitino/maintenance/scheduler/policy/expand/NearestPolicyTableEnumerator.java
+++
b/plugins/maintenance/src/main/java/org/apache/gravitino/maintenance/scheduler/policy/expand/NearestPolicyTableEnumerator.java
@@ -21,24 +21,35 @@ package
org.apache.gravitino.maintenance.scheduler.policy.expand;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Supplier;
+import javax.annotation.Nullable;
import org.apache.gravitino.Entity;
import org.apache.gravitino.GravitinoEnv;
import org.apache.gravitino.MetadataObject;
import org.apache.gravitino.MetadataObjects;
import org.apache.gravitino.NameIdentifier;
+import org.apache.gravitino.Namespace;
import org.apache.gravitino.RelationalEntity;
+import org.apache.gravitino.catalog.EntityCombinedTable;
+import org.apache.gravitino.catalog.SchemaDispatcher;
+import org.apache.gravitino.catalog.TableDispatcher;
+import org.apache.gravitino.exceptions.NoSuchCatalogException;
+import org.apache.gravitino.exceptions.NoSuchSchemaException;
+import org.apache.gravitino.exceptions.NoSuchTableException;
import org.apache.gravitino.meta.GenericEntity;
import org.apache.gravitino.meta.PolicyEntity;
import org.apache.gravitino.meta.SchemaEntity;
import org.apache.gravitino.meta.TableEntity;
import org.apache.gravitino.meta.TagEntity;
import org.apache.gravitino.policy.Policy;
+import org.apache.gravitino.rel.Table;
import org.apache.gravitino.storage.relational.service.PolicyTagRelService;
import org.apache.gravitino.storage.relational.service.SchemaMetaService;
import org.apache.gravitino.storage.relational.service.TableMetaService;
@@ -50,13 +61,16 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * Lists tables for which {@code policy} is the nearest maintenance policy of
its type.
+ * Finds tables where {@code policy} is the nearest maintenance policy of its
type.
*
- * <p>Walks {@code policy → tags → metadata objects}, expands catalog/schema
attachments to tables
- * (including tables in hierarchical descendant schemas), then keeps a table
only when {@link
- * NearestMaintenancePolicyResolver} selects this policy id and {@link
ExpandOccupancyGate} allows
- * enqueue. Scanning continues until {@code limit} enqueueable tables are
found (cooldown /
- * in-flight winners do not consume the batch budget).
+ * <p>Flow: {@code policy → tags → tagged objects}. Catalog/schema tags expand
to tables (including
+ * hierarchical child schemas). A table is kept only when {@link
NearestMaintenancePolicyResolver}
+ * picks this policy and {@link ExpandOccupancyGate} allows enqueue. Stops
after {@code limit}
+ * accepted tables (skipped cooldown / in-flight tables do not count).
+ *
+ * <p>Catalog/schema expansion prefers live {@code listSchemas}/{@code
listTables}, then imports
+ * only missing entities with {@code loadSchema}/{@code loadTable}. On list
failure, falls back to
+ * the relational store. A direct table tag uses the stored table id as-is.
*/
public final class NearestPolicyTableEnumerator implements
PolicyTableEnumerator {
@@ -67,23 +81,35 @@ public final class NearestPolicyTableEnumerator implements
PolicyTableEnumerator
private final SchemaMetaService schemaMetaService;
private final TableMetaService tableMetaService;
private final Supplier<NearestMaintenancePolicyResolver>
nearestResolverSupplier;
+ @Nullable private final SchemaDispatcher schemaDispatcher;
+ @Nullable private final TableDispatcher tableDispatcher;
- /** Uses relational singletons and {@link GravitinoEnv#entityStore()}. */
+ /**
+ * Production wiring via relational singletons and {@link GravitinoEnv}.
+ *
+ * <p>Resolves {@link GravitinoEnv#entityStore()} lazily so scheduler pool
startup can construct
+ * this enumerator before the env is fully initialized (unit tests).
Dispatchers may be null until
+ * env init; expand then falls back to relational meta listing.
+ */
public NearestPolicyTableEnumerator() {
this(
PolicyTagRelService.getInstance(),
TagMetaService.getInstance(),
SchemaMetaService.getInstance(),
TableMetaService.getInstance(),
- () -> new
NearestMaintenancePolicyResolver(GravitinoEnv.getInstance().entityStore()));
+ () -> new
NearestMaintenancePolicyResolver(GravitinoEnv.getInstance().entityStore()),
+ GravitinoEnv.getInstance().internalSchemaDispatcher(),
+ GravitinoEnv.getInstance().internalTableDispatcher());
}
/**
- * @param policyTagRelService policy-tag relations
- * @param tagMetaService tag metadata
- * @param schemaMetaService schema listing
- * @param tableMetaService table listing
- * @param nearestResolverSupplier nearest maintenance-policy resolver
+ * Meta-only constructor for unit tests (no catalog list/import).
+ *
+ * @param policyTagRelService policy ↔ tag edges
+ * @param tagMetaService tagged metadata objects
+ * @param schemaMetaService schemas already in the entity store
+ * @param tableMetaService tables already in the entity store
+ * @param nearestResolverSupplier decides which policy wins for a table
*/
public NearestPolicyTableEnumerator(
PolicyTagRelService policyTagRelService,
@@ -91,11 +117,40 @@ public final class NearestPolicyTableEnumerator implements
PolicyTableEnumerator
SchemaMetaService schemaMetaService,
TableMetaService tableMetaService,
Supplier<NearestMaintenancePolicyResolver> nearestResolverSupplier) {
+ this(
+ policyTagRelService,
+ tagMetaService,
+ schemaMetaService,
+ tableMetaService,
+ nearestResolverSupplier,
+ null,
+ null);
+ }
+
+ /**
+ * @param policyTagRelService policy ↔ tag edges
+ * @param tagMetaService tagged metadata objects
+ * @param schemaMetaService schemas already in the entity store
+ * @param tableMetaService tables already in the entity store
+ * @param nearestResolverSupplier decides which policy wins for a table
+ * @param schemaDispatcher optional live catalog schema list/import
+ * @param tableDispatcher optional live catalog table list/import
+ */
+ public NearestPolicyTableEnumerator(
+ PolicyTagRelService policyTagRelService,
+ TagMetaService tagMetaService,
+ SchemaMetaService schemaMetaService,
+ TableMetaService tableMetaService,
+ Supplier<NearestMaintenancePolicyResolver> nearestResolverSupplier,
+ @Nullable SchemaDispatcher schemaDispatcher,
+ @Nullable TableDispatcher tableDispatcher) {
this.policyTagRelService = policyTagRelService;
this.tagMetaService = tagMetaService;
this.schemaMetaService = schemaMetaService;
this.tableMetaService = tableMetaService;
this.nearestResolverSupplier = nearestResolverSupplier;
+ this.schemaDispatcher = schemaDispatcher;
+ this.tableDispatcher = tableDispatcher;
}
@Override
@@ -104,207 +159,409 @@ public final class NearestPolicyTableEnumerator
implements PolicyTableEnumerator
if (limit <= 0) {
return List.of();
}
- Policy.BuiltInType policyType = policy.policyType();
+
String metalake = policy.namespace().level(0);
NameIdentifier policyIdent = NameIdentifierUtil.ofPolicy(metalake,
policy.name());
- List<RelationalEntity<?>> tagEdges =
+ List<RelationalEntity<?>> policyTagEdges =
policyTagRelService.listRelations(List.of(policyIdent),
Entity.EntityType.POLICY);
- NearestMaintenancePolicyResolver nearest = nearestResolverSupplier.get();
- Set<Long> seenTables = new LinkedHashSet<>();
- Map<Long, ExpandTableCandidate> enqueueable = new LinkedHashMap<>();
- long nowMs = System.currentTimeMillis();
- ScanContext ctx =
- new ScanContext(
+ ScanState scan =
+ new ScanState(
metalake,
policy.id(),
- policyType,
- nearest,
+ policy.policyType(),
+ nearestResolverSupplier.get(),
occupancyGate,
minIntervalMs,
- nowMs,
- seenTables,
- enqueueable,
+ System.currentTimeMillis(),
+ new LinkedHashSet<>(),
+ new LinkedHashMap<>(),
limit);
- for (RelationalEntity<?> edge : tagEdges) {
+ for (RelationalEntity<?> edge : policyTagEdges) {
Object target = edge.targetEntity();
if (!(target instanceof TagEntity)) {
continue;
}
TagEntity tag = (TagEntity) target;
- List<GenericEntity> objects;
+ List<GenericEntity> taggedObjects;
try {
- objects =
tagMetaService.listAssociatedMetadataObjectsForTag(tag.nameIdentifier());
+ taggedObjects =
tagMetaService.listAssociatedMetadataObjectsForTag(tag.nameIdentifier());
} catch (IOException e) {
throw new UncheckedIOException("Failed to list objects for tag " +
tag.nameIdentifier(), e);
}
- for (GenericEntity object : objects) {
- if (enqueueable.size() >= limit) {
- return new ArrayList<>(enqueueable.values());
+ for (GenericEntity taggedObject : taggedObjects) {
+ if (scan.batchFull()) {
+ return new ArrayList<>(scan.acceptedByTableId.values());
}
- collectCandidates(object, ctx);
+ expandTaggedObject(taggedObject, scan);
}
}
- return new ArrayList<>(enqueueable.values());
+ return new ArrayList<>(scan.acceptedByTableId.values());
}
- private void collectCandidates(GenericEntity object, ScanContext ctx) {
+ /** Expands one tagged catalog/schema/table into candidate tables. */
+ private void expandTaggedObject(GenericEntity taggedObject, ScanState scan) {
MetadataObject.Type objectType;
try {
- objectType = MetadataObject.Type.valueOf(object.type().name());
+ objectType = MetadataObject.Type.valueOf(taggedObject.type().name());
} catch (IllegalArgumentException e) {
return;
}
- if (object.name() == null || object.name().isBlank()) {
+ if (taggedObject.name() == null || taggedObject.name().isBlank()) {
return;
}
+
MetadataObject metadataObject;
try {
- metadataObject = MetadataObjects.parse(object.name(), objectType);
+ metadataObject = MetadataObjects.parse(taggedObject.name(), objectType);
} catch (RuntimeException e) {
return;
}
switch (objectType) {
case TABLE:
- if (object.id() == null) {
- return;
+ if (taggedObject.id() != null) {
+ tryEnqueueTable(taggedObject.id(), metadataObject, scan);
}
- considerTable(object.id(), metadataObject, ctx);
break;
case SCHEMA:
- // A tag on a parent schema also covers tables in child schemas such
as {@code a:b}.
- expandSchema(metadataObject, ctx, true);
+ // Parent schema tags also cover hierarchical children (e.g. a → a:b).
+ expandSchema(metadataObject, scan, /*includeChildSchemas=*/ true);
break;
case CATALOG:
- expandCatalog(metadataObject, ctx);
+ expandCatalog(metadataObject, scan);
break;
default:
break;
}
}
- private void expandCatalog(MetadataObject catalog, ScanContext ctx) {
- List<SchemaEntity> schemas =
+ /** Lists schemas under a catalog tag, imports missing ones, then expands
each to tables. */
+ private void expandCatalog(MetadataObject catalog, ScanState scan) {
+ if (schemaDispatcher != null) {
+ try {
+ NameIdentifier[] liveSchemas =
+ schemaDispatcher.listSchemas(NamespaceUtil.ofSchema(scan.metalake,
catalog.name()));
+ Set<String> storedSchemaNames = storedSchemaNames(scan.metalake,
catalog.name());
+ for (NameIdentifier schemaIdent : liveSchemas) {
+ if (scan.batchFull()) {
+ return;
+ }
+ if (!importSchemaIfMissing(schemaIdent, storedSchemaNames)) {
+ continue;
+ }
+ expandSchema(
+ MetadataObjects.of(catalog.name(), schemaIdent.name(),
MetadataObject.Type.SCHEMA),
+ scan,
+ false);
+ }
+ return;
+ } catch (NoSuchCatalogException e) {
+ LOG.warn(
+ "Catalog {} missing while expanding policy attachment; falling
back to meta list: {}",
+ catalog.fullName(),
+ e.toString());
+ } catch (RuntimeException e) {
+ LOG.warn(
+ "Failed to list schemas from catalog {} for policy expand; falling
back to meta: {}",
+ catalog.fullName(),
+ e.toString());
+ }
+ }
+
+ for (SchemaEntity schema :
schemaMetaService.listSchemasByNamespace(
- NamespaceUtil.ofSchema(ctx.metalake, catalog.name()));
- for (SchemaEntity schema : schemas) {
- if (ctx.enqueueable.size() >= ctx.limit) {
+ NamespaceUtil.ofSchema(scan.metalake, catalog.name()))) {
+ if (scan.batchFull()) {
return;
}
- MetadataObject schemaObject =
- MetadataObjects.of(catalog.name(), schema.name(),
MetadataObject.Type.SCHEMA);
- // Catalog listing already includes every hierarchical schema name.
- expandSchema(schemaObject, ctx, false);
+ expandSchema(
+ MetadataObjects.of(catalog.name(), schema.name(),
MetadataObject.Type.SCHEMA),
+ scan,
+ false);
}
}
- private void expandSchema(MetadataObject schema, ScanContext ctx, boolean
includeDescendants) {
+ /**
+ * Expands tables in {@code schema}, and optionally hierarchical child
schemas under it.
+ *
+ * @param includeChildSchemas when true (schema tag), also walk descendants
such as {@code a:b}
+ */
+ private void expandSchema(MetadataObject schema, ScanState scan, boolean
includeChildSchemas) {
String catalogName = schema.parent();
if (catalogName == null || schema.name() == null ||
schema.name().isBlank()) {
return;
}
- expandTables(catalogName, schema.name(), ctx);
- if (!includeDescendants || ctx.enqueueable.size() >= ctx.limit) {
+
+ expandTablesInSchema(catalogName, schema.name(), scan);
+ if (!includeChildSchemas || scan.batchFull()) {
return;
}
- String separator = HierarchicalSchemaUtil.schemaSeparator();
- String descendantPrefix = schema.name() + separator;
- List<SchemaEntity> schemas =
-
schemaMetaService.listSchemasByNamespace(NamespaceUtil.ofSchema(ctx.metalake,
catalogName));
- for (SchemaEntity descendant : schemas) {
- if (ctx.enqueueable.size() >= ctx.limit) {
+
+ String childPrefix = schema.name() +
HierarchicalSchemaUtil.schemaSeparator();
+ if (schemaDispatcher != null) {
+ try {
+ NameIdentifier[] liveSchemas =
+ schemaDispatcher.listSchemas(NamespaceUtil.ofSchema(scan.metalake,
catalogName));
+ Set<String> storedSchemaNames = storedSchemaNames(scan.metalake,
catalogName);
+ for (NameIdentifier childSchema : liveSchemas) {
+ if (scan.batchFull()) {
+ return;
+ }
+ if (childSchema.name() == null ||
!childSchema.name().startsWith(childPrefix)) {
+ continue;
+ }
+ if (!importSchemaIfMissing(childSchema, storedSchemaNames)) {
+ continue;
+ }
+ expandTablesInSchema(catalogName, childSchema.name(), scan);
+ }
+ return;
+ } catch (RuntimeException e) {
+ LOG.warn(
+ "Failed to list child schemas under {} from catalog; falling back
to meta: {}",
+ schema.fullName(),
+ e.toString());
+ }
+ }
+
+ for (SchemaEntity child :
+ schemaMetaService.listSchemasByNamespace(
+ NamespaceUtil.ofSchema(scan.metalake, catalogName))) {
+ if (scan.batchFull()) {
return;
}
- if (descendant.name() == null ||
!descendant.name().startsWith(descendantPrefix)) {
+ if (child.name() == null || !child.name().startsWith(childPrefix)) {
continue;
}
- expandTables(catalogName, descendant.name(), ctx);
+ expandTablesInSchema(catalogName, child.name(), scan);
}
}
- private void expandTables(String catalogName, String schemaName, ScanContext
ctx) {
- List<TableEntity> tables =
- tableMetaService.listTablesByNamespace(
- NamespaceUtil.ofTable(ctx.metalake, catalogName, schemaName));
- for (TableEntity table : tables) {
- if (ctx.enqueueable.size() >= ctx.limit) {
+ /** Lists tables in one schema: live catalog first, entity store on failure.
*/
+ private void expandTablesInSchema(String catalogName, String schemaName,
ScanState scan) {
+ Namespace tableNamespace = NamespaceUtil.ofTable(scan.metalake,
catalogName, schemaName);
+ if (tableDispatcher != null) {
+ try {
+ NameIdentifier[] liveTables =
tableDispatcher.listTables(tableNamespace);
+ Map<String, TableEntity> storedTables =
storedTablesByName(tableNamespace);
+ for (NameIdentifier tableIdent : liveTables) {
+ if (scan.batchFull()) {
+ return;
+ }
+ if (!importTableIfMissing(tableIdent, storedTables)) {
+ continue;
+ }
+ tryEnqueueTable(
+ storedTables.get(tableIdent.name()).id(),
+ NameIdentifierUtil.toMetadataObject(tableIdent,
Entity.EntityType.TABLE),
+ scan);
+ }
+ return;
+ } catch (NoSuchSchemaException e) {
+ LOG.warn(
+ "Schema {}.{}.{} missing for policy expand; falling back to meta:
{}",
+ scan.metalake,
+ catalogName,
+ schemaName,
+ e.toString());
+ } catch (RuntimeException e) {
+ LOG.warn(
+ "Failed to list tables from catalog for {}.{}.{}; falling back to
meta: {}",
+ scan.metalake,
+ catalogName,
+ schemaName,
+ e.toString());
+ }
+ }
+
+ for (TableEntity table :
tableMetaService.listTablesByNamespace(tableNamespace)) {
+ if (scan.batchFull()) {
return;
}
- MetadataObject tableObject =
- NameIdentifierUtil.toMetadataObject(table.nameIdentifier(),
Entity.EntityType.TABLE);
- considerTable(table.id(), tableObject, ctx);
+ tryEnqueueTable(
+ table.id(),
+ NameIdentifierUtil.toMetadataObject(table.nameIdentifier(),
Entity.EntityType.TABLE),
+ scan);
}
}
- private static void considerTable(long tableId, MetadataObject tableObject,
ScanContext ctx) {
- if (tableId <= 0 || !ctx.seenTables.add(tableId)) {
+ /**
+ * Ensures {@code schemaIdent} exists in the entity store.
+ *
+ * @return {@code true} if already stored or import succeeded; {@code false}
if import failed
+ */
+ private boolean importSchemaIfMissing(NameIdentifier schemaIdent,
Set<String> storedSchemaNames) {
+ if (storedSchemaNames.contains(schemaIdent.name())) {
+ return true;
+ }
+ if (schemaDispatcher == null) {
+ return false;
+ }
+ try {
+ schemaDispatcher.loadSchema(schemaIdent);
+ storedSchemaNames.add(schemaIdent.name());
+ return true;
+ } catch (RuntimeException e) {
+ LOG.warn(
+ "Skip schema {} while importing missing entity for policy expand:
{}",
+ schemaIdent,
+ e.toString());
+ return false;
+ }
+ }
+
+ /**
+ * Ensures {@code tableIdent} exists in the entity store and records it in
{@code storedTables}.
+ *
+ * @return {@code true} if already stored or import succeeded; {@code false}
if import failed
+ */
+ private boolean importTableIfMissing(
+ NameIdentifier tableIdent, Map<String, TableEntity> storedTables) {
+ if (storedTables.containsKey(tableIdent.name())) {
+ return true;
+ }
+ if (tableDispatcher == null) {
+ return false;
+ }
+ try {
+ Table loaded = tableDispatcher.loadTable(tableIdent);
+ TableEntity entity = resolveStoredTable(loaded, tableIdent);
+ if (entity == null || entity.id() == null) {
+ LOG.warn("Skip table {} after import load: missing entity id",
tableIdent);
+ return false;
+ }
+ storedTables.put(tableIdent.name(), entity);
+ return true;
+ } catch (NoSuchTableException e) {
+ LOG.debug("Table {} disappeared during expand: {}", tableIdent,
e.toString());
+ return false;
+ } catch (RuntimeException e) {
+ LOG.warn("Skip table {} during catalog expand import: {}", tableIdent,
e.toString());
+ return false;
+ }
+ }
+
+ private Set<String> storedSchemaNames(String metalake, String catalogName) {
+ Set<String> names = new HashSet<>();
+ for (SchemaEntity schema :
+
schemaMetaService.listSchemasByNamespace(NamespaceUtil.ofSchema(metalake,
catalogName))) {
+ if (schema.name() != null) {
+ names.add(schema.name());
+ }
+ }
+ return names;
+ }
+
+ private Map<String, TableEntity> storedTablesByName(Namespace
tableNamespace) {
+ Map<String, TableEntity> byName = new HashMap<>();
+ for (TableEntity table :
tableMetaService.listTablesByNamespace(tableNamespace)) {
+ if (table.name() != null) {
+ byName.put(table.name(), table);
+ }
+ }
+ return byName;
+ }
+
+ /** Prefer the entity on {@link EntityCombinedTable}; otherwise look up the
store. */
+ @Nullable
+ private TableEntity resolveStoredTable(Table table, NameIdentifier
tableIdent) {
+ if (table instanceof EntityCombinedTable) {
+ TableEntity entity = ((EntityCombinedTable) table).tableFromGravitino();
+ if (entity != null && entity.id() != null) {
+ return entity;
+ }
+ }
+ try {
+ for (TableEntity entity :
tableMetaService.listTablesByNamespace(tableIdent.namespace())) {
+ if (tableIdent.name().equals(entity.name())) {
+ return entity;
+ }
+ }
+ } catch (RuntimeException e) {
+ LOG.warn("Failed to resolve table entity for {}: {}", tableIdent,
e.toString());
+ }
+ return null;
+ }
+
+ /**
+ * Accepts {@code tableId} into the batch when this policy is the nearest
winner and occupancy
+ * allows enqueue.
+ */
+ private static void tryEnqueueTable(long tableId, MetadataObject
tableObject, ScanState scan) {
+ if (tableId <= 0 || !scan.visitedTableIds.add(tableId)) {
return;
}
+
NearestMaintenancePolicyResolver.Decision decision =
- ctx.nearest.resolve(ctx.metalake, tableObject, ctx.policyType);
+ scan.nearestResolver.resolve(scan.metalake, tableObject,
scan.policyType);
if (decision.kind() ==
NearestMaintenancePolicyResolver.Decision.Kind.CONFLICT) {
LOG.error(
"Skip table {} for maintenance type {}: ambiguous policies {} on
nearest level {} ({})",
tableObject.fullName(),
- ctx.policyType,
+ scan.policyType,
decision.conflictingPolicyIds(),
decision.conflictLevel() == null ? "?" :
decision.conflictLevel().type(),
decision.conflictLevel() == null ? "?" :
decision.conflictLevel().fullName());
return;
}
- if (decision.winnerPolicyId().orElse(-1L) != ctx.policyId) {
+ if (decision.winnerPolicyId().orElse(-1L) != scan.policyId) {
return;
}
MetadataObject winnerLevel = decision.winnerLevel();
if (winnerLevel == null) {
return;
}
- if (ctx.occupancyGate.shouldSkip(tableId, ctx.policyId, ctx.minIntervalMs,
ctx.nowMs)) {
+ if (scan.occupancyGate.shouldSkip(tableId, scan.policyId,
scan.minIntervalMs, scan.nowMs)) {
LOG.debug(
"Skip cron-job enqueue for table {} policy {} (in-flight or within
minIntervalMs={})",
tableId,
- ctx.policyId,
- ctx.minIntervalMs);
+ scan.policyId,
+ scan.minIntervalMs);
return;
}
- ctx.enqueueable.put(tableId, new ExpandTableCandidate(tableId,
winnerLevel.type()));
+ scan.acceptedByTableId.put(tableId, new ExpandTableCandidate(tableId,
winnerLevel.type()));
}
- private static final class ScanContext {
+ /** Mutable progress for one {@link #listTableIds} call. */
+ private static final class ScanState {
private final String metalake;
private final long policyId;
private final Policy.BuiltInType policyType;
- private final NearestMaintenancePolicyResolver nearest;
+ private final NearestMaintenancePolicyResolver nearestResolver;
private final ExpandOccupancyGate occupancyGate;
private final long minIntervalMs;
private final long nowMs;
- private final Set<Long> seenTables;
- private final Map<Long, ExpandTableCandidate> enqueueable;
+ private final Set<Long> visitedTableIds;
+ private final Map<Long, ExpandTableCandidate> acceptedByTableId;
private final int limit;
- private ScanContext(
+ private ScanState(
String metalake,
long policyId,
Policy.BuiltInType policyType,
- NearestMaintenancePolicyResolver nearest,
+ NearestMaintenancePolicyResolver nearestResolver,
ExpandOccupancyGate occupancyGate,
long minIntervalMs,
long nowMs,
- Set<Long> seenTables,
- Map<Long, ExpandTableCandidate> enqueueable,
+ Set<Long> visitedTableIds,
+ Map<Long, ExpandTableCandidate> acceptedByTableId,
int limit) {
this.metalake = metalake;
this.policyId = policyId;
this.policyType = policyType;
- this.nearest = nearest;
+ this.nearestResolver = nearestResolver;
this.occupancyGate = occupancyGate;
this.minIntervalMs = minIntervalMs;
this.nowMs = nowMs;
- this.seenTables = seenTables;
- this.enqueueable = enqueueable;
+ this.visitedTableIds = visitedTableIds;
+ this.acceptedByTableId = acceptedByTableId;
this.limit = limit;
}
+
+ private boolean batchFull() {
+ return acceptedByTableId.size() >= limit;
+ }
}
}
diff --git
a/plugins/maintenance/src/test/java/org/apache/gravitino/maintenance/scheduler/policy/expand/TestNearestPolicyTableEnumerator.java
b/plugins/maintenance/src/test/java/org/apache/gravitino/maintenance/scheduler/policy/expand/TestNearestPolicyTableEnumerator.java
index f2ef37d454..cbcc510013 100644
---
a/plugins/maintenance/src/test/java/org/apache/gravitino/maintenance/scheduler/policy/expand/TestNearestPolicyTableEnumerator.java
+++
b/plugins/maintenance/src/test/java/org/apache/gravitino/maintenance/scheduler/policy/expand/TestNearestPolicyTableEnumerator.java
@@ -21,6 +21,8 @@ package
org.apache.gravitino.maintenance.scheduler.policy.expand;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.time.Instant;
@@ -28,8 +30,12 @@ import java.util.List;
import org.apache.gravitino.Entity;
import org.apache.gravitino.MetadataObject;
import org.apache.gravitino.MetadataObjects;
+import org.apache.gravitino.NameIdentifier;
import org.apache.gravitino.RelationalEntity;
import org.apache.gravitino.SupportsRelationOperations;
+import org.apache.gravitino.catalog.EntityCombinedTable;
+import org.apache.gravitino.catalog.SchemaDispatcher;
+import org.apache.gravitino.catalog.TableDispatcher;
import org.apache.gravitino.meta.AuditInfo;
import org.apache.gravitino.meta.GenericEntity;
import org.apache.gravitino.meta.PolicyEntity;
@@ -39,6 +45,7 @@ import org.apache.gravitino.meta.TagEntity;
import org.apache.gravitino.policy.IcebergDataCompactionContent;
import org.apache.gravitino.policy.Policy;
import org.apache.gravitino.policy.PolicyContents;
+import org.apache.gravitino.rel.Table;
import org.apache.gravitino.storage.relational.service.PolicyTagRelService;
import org.apache.gravitino.storage.relational.service.SchemaMetaService;
import org.apache.gravitino.storage.relational.service.TableMetaService;
@@ -49,6 +56,7 @@ import org.apache.gravitino.utils.NamespaceUtil;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
class TestNearestPolicyTableEnumerator {
@@ -59,6 +67,8 @@ class TestNearestPolicyTableEnumerator {
private SchemaMetaService schemaMetaService;
private TableMetaService tableMetaService;
private NearestMaintenancePolicyResolver nearest;
+ private SchemaDispatcher schemaDispatcher;
+ private TableDispatcher tableDispatcher;
private NearestPolicyTableEnumerator enumerator;
@BeforeEach
@@ -68,13 +78,22 @@ class TestNearestPolicyTableEnumerator {
schemaMetaService = mock(SchemaMetaService.class);
tableMetaService = mock(TableMetaService.class);
nearest = mock(NearestMaintenancePolicyResolver.class);
+ // Force meta fallback for tests that only stub SchemaMetaService /
TableMetaService.
+ schemaDispatcher = mock(SchemaDispatcher.class);
+ tableDispatcher = mock(TableDispatcher.class);
+ when(schemaDispatcher.listSchemas(any()))
+ .thenThrow(new RuntimeException("use meta listing in unit test"));
+ when(tableDispatcher.listTables(any()))
+ .thenThrow(new RuntimeException("use meta listing in unit test"));
enumerator =
new NearestPolicyTableEnumerator(
policyTagRelService,
tagMetaService,
schemaMetaService,
tableMetaService,
- () -> nearest);
+ () -> nearest,
+ schemaDispatcher,
+ tableDispatcher);
when(schemaMetaService.listSchemasByNamespace(any())).thenReturn(List.of());
}
@@ -289,6 +308,107 @@ class TestNearestPolicyTableEnumerator {
List.of(), tableIds(enumerator.listTableIds(policy, 10, allowAll(),
0L)));
}
+ @Test
+ void testImportsOnlyMissingTablesAfterCatalogList() throws Exception {
+ PolicyEntity policy = stubTaggedObject("catalog.schema",
Entity.EntityType.SCHEMA);
+ SchemaDispatcher schemas = mock(SchemaDispatcher.class);
+ TableDispatcher tables = mock(TableDispatcher.class);
+ NameIdentifier known = NameIdentifierUtil.ofTable(METALAKE, "catalog",
"schema", "known");
+ NameIdentifier missing = NameIdentifierUtil.ofTable(METALAKE, "catalog",
"schema", "missing");
+ when(tables.listTables(eq(NamespaceUtil.ofTable(METALAKE, "catalog",
"schema"))))
+ .thenReturn(new NameIdentifier[] {known, missing});
+ when(tableMetaService.listTablesByNamespace(
+ eq(NamespaceUtil.ofTable(METALAKE, "catalog", "schema"))))
+ .thenReturn(List.of(table(500L, "known")));
+ when(tables.loadTable(eq(missing)))
+ .thenReturn(
+ EntityCombinedTable.of(Mockito.mock(Table.class), table(501L,
"missing"))
+ .withImported(true));
+
+ MetadataObject schemaObj =
+ MetadataObjects.of(
+ java.util.Arrays.asList("catalog", "schema"),
MetadataObject.Type.SCHEMA);
+ stubWinner(tableMeta("catalog", "schema", "known"), schemaObj);
+ stubWinner(tableMeta("catalog", "schema", "missing"), schemaObj);
+
+ Assertions.assertEquals(
+ List.of(500L, 501L),
+ tableIds(catalogEnumerator(schemas, tables).listTableIds(policy, 10,
allowAll(), 0L)));
+ verify(tables).loadTable(missing);
+ verify(tables, never()).loadTable(known);
+ }
+
+ @Test
+ void testImportsOnlyMissingSchemasAfterCatalogList() throws Exception {
+ PolicyEntity policy = stubTaggedObject("catalog",
Entity.EntityType.CATALOG);
+ SchemaDispatcher schemas = mock(SchemaDispatcher.class);
+ TableDispatcher tables = mock(TableDispatcher.class);
+ NameIdentifier known = NameIdentifierUtil.ofSchema(METALAKE, "catalog",
"known");
+ NameIdentifier missing = NameIdentifierUtil.ofSchema(METALAKE, "catalog",
"missing");
+ when(schemas.listSchemas(eq(NamespaceUtil.ofSchema(METALAKE, "catalog"))))
+ .thenReturn(new NameIdentifier[] {known, missing});
+
when(schemaMetaService.listSchemasByNamespace(eq(NamespaceUtil.ofSchema(METALAKE,
"catalog"))))
+ .thenReturn(List.of(schema(2L, "known")));
+
when(schemas.loadSchema(eq(missing))).thenReturn(mock(org.apache.gravitino.Schema.class));
+ when(tables.listTables(any())).thenReturn(new NameIdentifier[0]);
+ when(tableMetaService.listTablesByNamespace(any())).thenReturn(List.of());
+
+ Assertions.assertEquals(
+ List.of(),
+ tableIds(catalogEnumerator(schemas, tables).listTableIds(policy, 10,
allowAll(), 0L)));
+ verify(schemas).loadSchema(missing);
+ verify(schemas, never()).loadSchema(known);
+ }
+
+ private PolicyEntity stubTaggedObject(String objectName, Entity.EntityType
type)
+ throws Exception {
+ PolicyEntity policy = policy(10L, "compaction");
+ TagEntity tag = tag(1L, "maint");
+ when(policyTagRelService.listRelations(
+ eq(List.of(NameIdentifierUtil.ofPolicy(METALAKE, "compaction"))),
+ eq(Entity.EntityType.POLICY)))
+ .thenReturn(List.of(policyTagEdge(policy, tag)));
+
when(tagMetaService.listAssociatedMetadataObjectsForTag(tag.nameIdentifier()))
+ .thenReturn(
+ List.of(
+ GenericEntity.builder()
+ .withId(1L)
+ .withName(objectName)
+ .withEntityType(type)
+ .build()));
+ return policy;
+ }
+
+ private void stubWinner(MetadataObject table, MetadataObject nearestObject) {
+ when(nearest.resolve(METALAKE, table,
Policy.BuiltInType.ICEBERG_COMPACTION))
+ .thenReturn(NearestMaintenancePolicyResolver.Decision.winner(10L,
nearestObject));
+ }
+
+ private NearestPolicyTableEnumerator catalogEnumerator(
+ SchemaDispatcher schemas, TableDispatcher tables) {
+ return new NearestPolicyTableEnumerator(
+ policyTagRelService,
+ tagMetaService,
+ schemaMetaService,
+ tableMetaService,
+ () -> nearest,
+ schemas,
+ tables);
+ }
+
+ private static MetadataObject tableMeta(String... parts) {
+ return MetadataObjects.of(java.util.Arrays.asList(parts),
MetadataObject.Type.TABLE);
+ }
+
+ private static TableEntity table(long id, String name) {
+ return TableEntity.builder()
+ .withId(id)
+ .withName(name)
+ .withNamespace(NamespaceUtil.ofTable(METALAKE, "catalog", "schema"))
+ .withAuditInfo(audit())
+ .build();
+ }
+
private static List<Long> tableIds(List<ExpandTableCandidate> candidates) {
return candidates.stream().map(ExpandTableCandidate::tableId).toList();
}