This is an automated email from the ASF dual-hosted git repository.

roryqi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git


The following commit(s) were added to refs/heads/main by this push:
     new e7c33c2471 [#14298] improvement(tms): Import only missing 
schemas/tables on policy expand (#14328)
e7c33c2471 is described below

commit e7c33c247142878d9928db7d276a212a9f87d209
Author: MaSai <[email protected]>
AuthorDate: Thu Oct 29 15:06:53 2026 +0800

    [#14298] improvement(tms): Import only missing schemas/tables on policy 
expand (#14328)
    
    ### What changes were proposed in this pull request?
    
    - Prefer live `listSchemas`/`listTables` when expanding catalog/schema
    tags in `NearestPolicyTableEnumerator`.
    - Import only missing schema/table entities into the relational store
    (instead of always loading everything).
    
    Split out of #14300 so the iceberg catalog-wrapper cache change stays
    separate from TMS enumerator work.
    
    ### Why are the changes needed?
    
    Policy expand should avoid redundant import/load work when
    schemas/tables already exist in entity store, while still discovering
    live tables from the catalog.
    
    Fixes #14298 (TMS portion)
    
    ### Does this PR introduce _any_ user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    - `TestNearestPolicyTableEnumerator`
    
    
    Made with [Cursor](https://cursor.com)
    
    ---------
    
    Co-authored-by: Cursor <[email protected]>
---
 .../expand/NearestPolicyTableEnumerator.java       | 443 ++++++++++++++++-----
 .../expand/TestNearestPolicyTableEnumerator.java   | 122 +++++-
 2 files changed, 471 insertions(+), 94 deletions(-)

diff --git 
a/plugins/maintenance/src/main/java/org/apache/gravitino/maintenance/scheduler/policy/expand/NearestPolicyTableEnumerator.java
 
b/plugins/maintenance/src/main/java/org/apache/gravitino/maintenance/scheduler/policy/expand/NearestPolicyTableEnumerator.java
index 35b5ee2d12..938e54767b 100644
--- 
a/plugins/maintenance/src/main/java/org/apache/gravitino/maintenance/scheduler/policy/expand/NearestPolicyTableEnumerator.java
+++ 
b/plugins/maintenance/src/main/java/org/apache/gravitino/maintenance/scheduler/policy/expand/NearestPolicyTableEnumerator.java
@@ -21,24 +21,35 @@ package 
org.apache.gravitino.maintenance.scheduler.policy.expand;
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.function.Supplier;
+import javax.annotation.Nullable;
 import org.apache.gravitino.Entity;
 import org.apache.gravitino.GravitinoEnv;
 import org.apache.gravitino.MetadataObject;
 import org.apache.gravitino.MetadataObjects;
 import org.apache.gravitino.NameIdentifier;
+import org.apache.gravitino.Namespace;
 import org.apache.gravitino.RelationalEntity;
+import org.apache.gravitino.catalog.EntityCombinedTable;
+import org.apache.gravitino.catalog.SchemaDispatcher;
+import org.apache.gravitino.catalog.TableDispatcher;
+import org.apache.gravitino.exceptions.NoSuchCatalogException;
+import org.apache.gravitino.exceptions.NoSuchSchemaException;
+import org.apache.gravitino.exceptions.NoSuchTableException;
 import org.apache.gravitino.meta.GenericEntity;
 import org.apache.gravitino.meta.PolicyEntity;
 import org.apache.gravitino.meta.SchemaEntity;
 import org.apache.gravitino.meta.TableEntity;
 import org.apache.gravitino.meta.TagEntity;
 import org.apache.gravitino.policy.Policy;
+import org.apache.gravitino.rel.Table;
 import org.apache.gravitino.storage.relational.service.PolicyTagRelService;
 import org.apache.gravitino.storage.relational.service.SchemaMetaService;
 import org.apache.gravitino.storage.relational.service.TableMetaService;
@@ -50,13 +61,16 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
- * Lists tables for which {@code policy} is the nearest maintenance policy of 
its type.
+ * Finds tables where {@code policy} is the nearest maintenance policy of its 
type.
  *
- * <p>Walks {@code policy → tags → metadata objects}, expands catalog/schema 
attachments to tables
- * (including tables in hierarchical descendant schemas), then keeps a table 
only when {@link
- * NearestMaintenancePolicyResolver} selects this policy id and {@link 
ExpandOccupancyGate} allows
- * enqueue. Scanning continues until {@code limit} enqueueable tables are 
found (cooldown /
- * in-flight winners do not consume the batch budget).
+ * <p>Flow: {@code policy → tags → tagged objects}. Catalog/schema tags expand 
to tables (including
+ * hierarchical child schemas). A table is kept only when {@link 
NearestMaintenancePolicyResolver}
+ * picks this policy and {@link ExpandOccupancyGate} allows enqueue. Stops 
after {@code limit}
+ * accepted tables (skipped cooldown / in-flight tables do not count).
+ *
+ * <p>Catalog/schema expansion prefers live {@code listSchemas}/{@code 
listTables}, then imports
+ * only missing entities with {@code loadSchema}/{@code loadTable}. On list 
failure, falls back to
+ * the relational store. A direct table tag uses the stored table id as-is.
  */
 public final class NearestPolicyTableEnumerator implements 
PolicyTableEnumerator {
 
@@ -67,23 +81,35 @@ public final class NearestPolicyTableEnumerator implements 
PolicyTableEnumerator
   private final SchemaMetaService schemaMetaService;
   private final TableMetaService tableMetaService;
   private final Supplier<NearestMaintenancePolicyResolver> 
nearestResolverSupplier;
+  @Nullable private final SchemaDispatcher schemaDispatcher;
+  @Nullable private final TableDispatcher tableDispatcher;
 
-  /** Uses relational singletons and {@link GravitinoEnv#entityStore()}. */
+  /**
+   * Production wiring via relational singletons and {@link GravitinoEnv}.
+   *
+   * <p>Resolves {@link GravitinoEnv#entityStore()} lazily so scheduler pool 
startup can construct
+   * this enumerator before the env is fully initialized (unit tests). 
Dispatchers may be null until
+   * env init; expand then falls back to relational meta listing.
+   */
   public NearestPolicyTableEnumerator() {
     this(
         PolicyTagRelService.getInstance(),
         TagMetaService.getInstance(),
         SchemaMetaService.getInstance(),
         TableMetaService.getInstance(),
-        () -> new 
NearestMaintenancePolicyResolver(GravitinoEnv.getInstance().entityStore()));
+        () -> new 
NearestMaintenancePolicyResolver(GravitinoEnv.getInstance().entityStore()),
+        GravitinoEnv.getInstance().internalSchemaDispatcher(),
+        GravitinoEnv.getInstance().internalTableDispatcher());
   }
 
   /**
-   * @param policyTagRelService policy-tag relations
-   * @param tagMetaService tag metadata
-   * @param schemaMetaService schema listing
-   * @param tableMetaService table listing
-   * @param nearestResolverSupplier nearest maintenance-policy resolver
+   * Meta-only constructor for unit tests (no catalog list/import).
+   *
+   * @param policyTagRelService policy ↔ tag edges
+   * @param tagMetaService tagged metadata objects
+   * @param schemaMetaService schemas already in the entity store
+   * @param tableMetaService tables already in the entity store
+   * @param nearestResolverSupplier decides which policy wins for a table
    */
   public NearestPolicyTableEnumerator(
       PolicyTagRelService policyTagRelService,
@@ -91,11 +117,40 @@ public final class NearestPolicyTableEnumerator implements 
PolicyTableEnumerator
       SchemaMetaService schemaMetaService,
       TableMetaService tableMetaService,
       Supplier<NearestMaintenancePolicyResolver> nearestResolverSupplier) {
+    this(
+        policyTagRelService,
+        tagMetaService,
+        schemaMetaService,
+        tableMetaService,
+        nearestResolverSupplier,
+        null,
+        null);
+  }
+
+  /**
+   * @param policyTagRelService policy ↔ tag edges
+   * @param tagMetaService tagged metadata objects
+   * @param schemaMetaService schemas already in the entity store
+   * @param tableMetaService tables already in the entity store
+   * @param nearestResolverSupplier decides which policy wins for a table
+   * @param schemaDispatcher optional live catalog schema list/import
+   * @param tableDispatcher optional live catalog table list/import
+   */
+  public NearestPolicyTableEnumerator(
+      PolicyTagRelService policyTagRelService,
+      TagMetaService tagMetaService,
+      SchemaMetaService schemaMetaService,
+      TableMetaService tableMetaService,
+      Supplier<NearestMaintenancePolicyResolver> nearestResolverSupplier,
+      @Nullable SchemaDispatcher schemaDispatcher,
+      @Nullable TableDispatcher tableDispatcher) {
     this.policyTagRelService = policyTagRelService;
     this.tagMetaService = tagMetaService;
     this.schemaMetaService = schemaMetaService;
     this.tableMetaService = tableMetaService;
     this.nearestResolverSupplier = nearestResolverSupplier;
+    this.schemaDispatcher = schemaDispatcher;
+    this.tableDispatcher = tableDispatcher;
   }
 
   @Override
@@ -104,207 +159,409 @@ public final class NearestPolicyTableEnumerator 
implements PolicyTableEnumerator
     if (limit <= 0) {
       return List.of();
     }
-    Policy.BuiltInType policyType = policy.policyType();
+
     String metalake = policy.namespace().level(0);
     NameIdentifier policyIdent = NameIdentifierUtil.ofPolicy(metalake, 
policy.name());
-    List<RelationalEntity<?>> tagEdges =
+    List<RelationalEntity<?>> policyTagEdges =
         policyTagRelService.listRelations(List.of(policyIdent), 
Entity.EntityType.POLICY);
 
-    NearestMaintenancePolicyResolver nearest = nearestResolverSupplier.get();
-    Set<Long> seenTables = new LinkedHashSet<>();
-    Map<Long, ExpandTableCandidate> enqueueable = new LinkedHashMap<>();
-    long nowMs = System.currentTimeMillis();
-    ScanContext ctx =
-        new ScanContext(
+    ScanState scan =
+        new ScanState(
             metalake,
             policy.id(),
-            policyType,
-            nearest,
+            policy.policyType(),
+            nearestResolverSupplier.get(),
             occupancyGate,
             minIntervalMs,
-            nowMs,
-            seenTables,
-            enqueueable,
+            System.currentTimeMillis(),
+            new LinkedHashSet<>(),
+            new LinkedHashMap<>(),
             limit);
 
-    for (RelationalEntity<?> edge : tagEdges) {
+    for (RelationalEntity<?> edge : policyTagEdges) {
       Object target = edge.targetEntity();
       if (!(target instanceof TagEntity)) {
         continue;
       }
       TagEntity tag = (TagEntity) target;
-      List<GenericEntity> objects;
+      List<GenericEntity> taggedObjects;
       try {
-        objects = 
tagMetaService.listAssociatedMetadataObjectsForTag(tag.nameIdentifier());
+        taggedObjects = 
tagMetaService.listAssociatedMetadataObjectsForTag(tag.nameIdentifier());
       } catch (IOException e) {
         throw new UncheckedIOException("Failed to list objects for tag " + 
tag.nameIdentifier(), e);
       }
-      for (GenericEntity object : objects) {
-        if (enqueueable.size() >= limit) {
-          return new ArrayList<>(enqueueable.values());
+      for (GenericEntity taggedObject : taggedObjects) {
+        if (scan.batchFull()) {
+          return new ArrayList<>(scan.acceptedByTableId.values());
         }
-        collectCandidates(object, ctx);
+        expandTaggedObject(taggedObject, scan);
       }
     }
-    return new ArrayList<>(enqueueable.values());
+    return new ArrayList<>(scan.acceptedByTableId.values());
   }
 
-  private void collectCandidates(GenericEntity object, ScanContext ctx) {
+  /** Expands one tagged catalog/schema/table into candidate tables. */
+  private void expandTaggedObject(GenericEntity taggedObject, ScanState scan) {
     MetadataObject.Type objectType;
     try {
-      objectType = MetadataObject.Type.valueOf(object.type().name());
+      objectType = MetadataObject.Type.valueOf(taggedObject.type().name());
     } catch (IllegalArgumentException e) {
       return;
     }
-    if (object.name() == null || object.name().isBlank()) {
+    if (taggedObject.name() == null || taggedObject.name().isBlank()) {
       return;
     }
+
     MetadataObject metadataObject;
     try {
-      metadataObject = MetadataObjects.parse(object.name(), objectType);
+      metadataObject = MetadataObjects.parse(taggedObject.name(), objectType);
     } catch (RuntimeException e) {
       return;
     }
 
     switch (objectType) {
       case TABLE:
-        if (object.id() == null) {
-          return;
+        if (taggedObject.id() != null) {
+          tryEnqueueTable(taggedObject.id(), metadataObject, scan);
         }
-        considerTable(object.id(), metadataObject, ctx);
         break;
       case SCHEMA:
-        // A tag on a parent schema also covers tables in child schemas such 
as {@code a:b}.
-        expandSchema(metadataObject, ctx, true);
+        // Parent schema tags also cover hierarchical children (e.g. a → a:b).
+        expandSchema(metadataObject, scan, /*includeChildSchemas=*/ true);
         break;
       case CATALOG:
-        expandCatalog(metadataObject, ctx);
+        expandCatalog(metadataObject, scan);
         break;
       default:
         break;
     }
   }
 
-  private void expandCatalog(MetadataObject catalog, ScanContext ctx) {
-    List<SchemaEntity> schemas =
+  /** Lists schemas under a catalog tag, imports missing ones, then expands 
each to tables. */
+  private void expandCatalog(MetadataObject catalog, ScanState scan) {
+    if (schemaDispatcher != null) {
+      try {
+        NameIdentifier[] liveSchemas =
+            schemaDispatcher.listSchemas(NamespaceUtil.ofSchema(scan.metalake, 
catalog.name()));
+        Set<String> storedSchemaNames = storedSchemaNames(scan.metalake, 
catalog.name());
+        for (NameIdentifier schemaIdent : liveSchemas) {
+          if (scan.batchFull()) {
+            return;
+          }
+          if (!importSchemaIfMissing(schemaIdent, storedSchemaNames)) {
+            continue;
+          }
+          expandSchema(
+              MetadataObjects.of(catalog.name(), schemaIdent.name(), 
MetadataObject.Type.SCHEMA),
+              scan,
+              false);
+        }
+        return;
+      } catch (NoSuchCatalogException e) {
+        LOG.warn(
+            "Catalog {} missing while expanding policy attachment; falling 
back to meta list: {}",
+            catalog.fullName(),
+            e.toString());
+      } catch (RuntimeException e) {
+        LOG.warn(
+            "Failed to list schemas from catalog {} for policy expand; falling 
back to meta: {}",
+            catalog.fullName(),
+            e.toString());
+      }
+    }
+
+    for (SchemaEntity schema :
         schemaMetaService.listSchemasByNamespace(
-            NamespaceUtil.ofSchema(ctx.metalake, catalog.name()));
-    for (SchemaEntity schema : schemas) {
-      if (ctx.enqueueable.size() >= ctx.limit) {
+            NamespaceUtil.ofSchema(scan.metalake, catalog.name()))) {
+      if (scan.batchFull()) {
         return;
       }
-      MetadataObject schemaObject =
-          MetadataObjects.of(catalog.name(), schema.name(), 
MetadataObject.Type.SCHEMA);
-      // Catalog listing already includes every hierarchical schema name.
-      expandSchema(schemaObject, ctx, false);
+      expandSchema(
+          MetadataObjects.of(catalog.name(), schema.name(), 
MetadataObject.Type.SCHEMA),
+          scan,
+          false);
     }
   }
 
-  private void expandSchema(MetadataObject schema, ScanContext ctx, boolean 
includeDescendants) {
+  /**
+   * Expands tables in {@code schema}, and optionally hierarchical child 
schemas under it.
+   *
+   * @param includeChildSchemas when true (schema tag), also walk descendants 
such as {@code a:b}
+   */
+  private void expandSchema(MetadataObject schema, ScanState scan, boolean 
includeChildSchemas) {
     String catalogName = schema.parent();
     if (catalogName == null || schema.name() == null || 
schema.name().isBlank()) {
       return;
     }
-    expandTables(catalogName, schema.name(), ctx);
-    if (!includeDescendants || ctx.enqueueable.size() >= ctx.limit) {
+
+    expandTablesInSchema(catalogName, schema.name(), scan);
+    if (!includeChildSchemas || scan.batchFull()) {
       return;
     }
-    String separator = HierarchicalSchemaUtil.schemaSeparator();
-    String descendantPrefix = schema.name() + separator;
-    List<SchemaEntity> schemas =
-        
schemaMetaService.listSchemasByNamespace(NamespaceUtil.ofSchema(ctx.metalake, 
catalogName));
-    for (SchemaEntity descendant : schemas) {
-      if (ctx.enqueueable.size() >= ctx.limit) {
+
+    String childPrefix = schema.name() + 
HierarchicalSchemaUtil.schemaSeparator();
+    if (schemaDispatcher != null) {
+      try {
+        NameIdentifier[] liveSchemas =
+            schemaDispatcher.listSchemas(NamespaceUtil.ofSchema(scan.metalake, 
catalogName));
+        Set<String> storedSchemaNames = storedSchemaNames(scan.metalake, 
catalogName);
+        for (NameIdentifier childSchema : liveSchemas) {
+          if (scan.batchFull()) {
+            return;
+          }
+          if (childSchema.name() == null || 
!childSchema.name().startsWith(childPrefix)) {
+            continue;
+          }
+          if (!importSchemaIfMissing(childSchema, storedSchemaNames)) {
+            continue;
+          }
+          expandTablesInSchema(catalogName, childSchema.name(), scan);
+        }
+        return;
+      } catch (RuntimeException e) {
+        LOG.warn(
+            "Failed to list child schemas under {} from catalog; falling back 
to meta: {}",
+            schema.fullName(),
+            e.toString());
+      }
+    }
+
+    for (SchemaEntity child :
+        schemaMetaService.listSchemasByNamespace(
+            NamespaceUtil.ofSchema(scan.metalake, catalogName))) {
+      if (scan.batchFull()) {
         return;
       }
-      if (descendant.name() == null || 
!descendant.name().startsWith(descendantPrefix)) {
+      if (child.name() == null || !child.name().startsWith(childPrefix)) {
         continue;
       }
-      expandTables(catalogName, descendant.name(), ctx);
+      expandTablesInSchema(catalogName, child.name(), scan);
     }
   }
 
-  private void expandTables(String catalogName, String schemaName, ScanContext 
ctx) {
-    List<TableEntity> tables =
-        tableMetaService.listTablesByNamespace(
-            NamespaceUtil.ofTable(ctx.metalake, catalogName, schemaName));
-    for (TableEntity table : tables) {
-      if (ctx.enqueueable.size() >= ctx.limit) {
+  /** Lists tables in one schema: live catalog first, entity store on failure. 
*/
+  private void expandTablesInSchema(String catalogName, String schemaName, 
ScanState scan) {
+    Namespace tableNamespace = NamespaceUtil.ofTable(scan.metalake, 
catalogName, schemaName);
+    if (tableDispatcher != null) {
+      try {
+        NameIdentifier[] liveTables = 
tableDispatcher.listTables(tableNamespace);
+        Map<String, TableEntity> storedTables = 
storedTablesByName(tableNamespace);
+        for (NameIdentifier tableIdent : liveTables) {
+          if (scan.batchFull()) {
+            return;
+          }
+          if (!importTableIfMissing(tableIdent, storedTables)) {
+            continue;
+          }
+          tryEnqueueTable(
+              storedTables.get(tableIdent.name()).id(),
+              NameIdentifierUtil.toMetadataObject(tableIdent, 
Entity.EntityType.TABLE),
+              scan);
+        }
+        return;
+      } catch (NoSuchSchemaException e) {
+        LOG.warn(
+            "Schema {}.{}.{} missing for policy expand; falling back to meta: 
{}",
+            scan.metalake,
+            catalogName,
+            schemaName,
+            e.toString());
+      } catch (RuntimeException e) {
+        LOG.warn(
+            "Failed to list tables from catalog for {}.{}.{}; falling back to 
meta: {}",
+            scan.metalake,
+            catalogName,
+            schemaName,
+            e.toString());
+      }
+    }
+
+    for (TableEntity table : 
tableMetaService.listTablesByNamespace(tableNamespace)) {
+      if (scan.batchFull()) {
         return;
       }
-      MetadataObject tableObject =
-          NameIdentifierUtil.toMetadataObject(table.nameIdentifier(), 
Entity.EntityType.TABLE);
-      considerTable(table.id(), tableObject, ctx);
+      tryEnqueueTable(
+          table.id(),
+          NameIdentifierUtil.toMetadataObject(table.nameIdentifier(), 
Entity.EntityType.TABLE),
+          scan);
     }
   }
 
-  private static void considerTable(long tableId, MetadataObject tableObject, 
ScanContext ctx) {
-    if (tableId <= 0 || !ctx.seenTables.add(tableId)) {
+  /**
+   * Ensures {@code schemaIdent} exists in the entity store.
+   *
+   * @return {@code true} if already stored or import succeeded; {@code false} 
if import failed
+   */
+  private boolean importSchemaIfMissing(NameIdentifier schemaIdent, 
Set<String> storedSchemaNames) {
+    if (storedSchemaNames.contains(schemaIdent.name())) {
+      return true;
+    }
+    if (schemaDispatcher == null) {
+      return false;
+    }
+    try {
+      schemaDispatcher.loadSchema(schemaIdent);
+      storedSchemaNames.add(schemaIdent.name());
+      return true;
+    } catch (RuntimeException e) {
+      LOG.warn(
+          "Skip schema {} while importing missing entity for policy expand: 
{}",
+          schemaIdent,
+          e.toString());
+      return false;
+    }
+  }
+
+  /**
+   * Ensures {@code tableIdent} exists in the entity store and records it in 
{@code storedTables}.
+   *
+   * @return {@code true} if already stored or import succeeded; {@code false} 
if import failed
+   */
+  private boolean importTableIfMissing(
+      NameIdentifier tableIdent, Map<String, TableEntity> storedTables) {
+    if (storedTables.containsKey(tableIdent.name())) {
+      return true;
+    }
+    if (tableDispatcher == null) {
+      return false;
+    }
+    try {
+      Table loaded = tableDispatcher.loadTable(tableIdent);
+      TableEntity entity = resolveStoredTable(loaded, tableIdent);
+      if (entity == null || entity.id() == null) {
+        LOG.warn("Skip table {} after import load: missing entity id", 
tableIdent);
+        return false;
+      }
+      storedTables.put(tableIdent.name(), entity);
+      return true;
+    } catch (NoSuchTableException e) {
+      LOG.debug("Table {} disappeared during expand: {}", tableIdent, 
e.toString());
+      return false;
+    } catch (RuntimeException e) {
+      LOG.warn("Skip table {} during catalog expand import: {}", tableIdent, 
e.toString());
+      return false;
+    }
+  }
+
+  private Set<String> storedSchemaNames(String metalake, String catalogName) {
+    Set<String> names = new HashSet<>();
+    for (SchemaEntity schema :
+        
schemaMetaService.listSchemasByNamespace(NamespaceUtil.ofSchema(metalake, 
catalogName))) {
+      if (schema.name() != null) {
+        names.add(schema.name());
+      }
+    }
+    return names;
+  }
+
+  private Map<String, TableEntity> storedTablesByName(Namespace 
tableNamespace) {
+    Map<String, TableEntity> byName = new HashMap<>();
+    for (TableEntity table : 
tableMetaService.listTablesByNamespace(tableNamespace)) {
+      if (table.name() != null) {
+        byName.put(table.name(), table);
+      }
+    }
+    return byName;
+  }
+
+  /** Prefer the entity on {@link EntityCombinedTable}; otherwise look up the 
store. */
+  @Nullable
+  private TableEntity resolveStoredTable(Table table, NameIdentifier 
tableIdent) {
+    if (table instanceof EntityCombinedTable) {
+      TableEntity entity = ((EntityCombinedTable) table).tableFromGravitino();
+      if (entity != null && entity.id() != null) {
+        return entity;
+      }
+    }
+    try {
+      for (TableEntity entity : 
tableMetaService.listTablesByNamespace(tableIdent.namespace())) {
+        if (tableIdent.name().equals(entity.name())) {
+          return entity;
+        }
+      }
+    } catch (RuntimeException e) {
+      LOG.warn("Failed to resolve table entity for {}: {}", tableIdent, 
e.toString());
+    }
+    return null;
+  }
+
+  /**
+   * Accepts {@code tableId} into the batch when this policy is the nearest 
winner and occupancy
+   * allows enqueue.
+   */
+  private static void tryEnqueueTable(long tableId, MetadataObject 
tableObject, ScanState scan) {
+    if (tableId <= 0 || !scan.visitedTableIds.add(tableId)) {
       return;
     }
+
     NearestMaintenancePolicyResolver.Decision decision =
-        ctx.nearest.resolve(ctx.metalake, tableObject, ctx.policyType);
+        scan.nearestResolver.resolve(scan.metalake, tableObject, 
scan.policyType);
     if (decision.kind() == 
NearestMaintenancePolicyResolver.Decision.Kind.CONFLICT) {
       LOG.error(
           "Skip table {} for maintenance type {}: ambiguous policies {} on 
nearest level {} ({})",
           tableObject.fullName(),
-          ctx.policyType,
+          scan.policyType,
           decision.conflictingPolicyIds(),
           decision.conflictLevel() == null ? "?" : 
decision.conflictLevel().type(),
           decision.conflictLevel() == null ? "?" : 
decision.conflictLevel().fullName());
       return;
     }
-    if (decision.winnerPolicyId().orElse(-1L) != ctx.policyId) {
+    if (decision.winnerPolicyId().orElse(-1L) != scan.policyId) {
       return;
     }
     MetadataObject winnerLevel = decision.winnerLevel();
     if (winnerLevel == null) {
       return;
     }
-    if (ctx.occupancyGate.shouldSkip(tableId, ctx.policyId, ctx.minIntervalMs, 
ctx.nowMs)) {
+    if (scan.occupancyGate.shouldSkip(tableId, scan.policyId, 
scan.minIntervalMs, scan.nowMs)) {
       LOG.debug(
           "Skip cron-job enqueue for table {} policy {} (in-flight or within 
minIntervalMs={})",
           tableId,
-          ctx.policyId,
-          ctx.minIntervalMs);
+          scan.policyId,
+          scan.minIntervalMs);
       return;
     }
-    ctx.enqueueable.put(tableId, new ExpandTableCandidate(tableId, 
winnerLevel.type()));
+    scan.acceptedByTableId.put(tableId, new ExpandTableCandidate(tableId, 
winnerLevel.type()));
   }
 
-  private static final class ScanContext {
+  /** Mutable progress for one {@link #listTableIds} call. */
+  private static final class ScanState {
     private final String metalake;
     private final long policyId;
     private final Policy.BuiltInType policyType;
-    private final NearestMaintenancePolicyResolver nearest;
+    private final NearestMaintenancePolicyResolver nearestResolver;
     private final ExpandOccupancyGate occupancyGate;
     private final long minIntervalMs;
     private final long nowMs;
-    private final Set<Long> seenTables;
-    private final Map<Long, ExpandTableCandidate> enqueueable;
+    private final Set<Long> visitedTableIds;
+    private final Map<Long, ExpandTableCandidate> acceptedByTableId;
     private final int limit;
 
-    private ScanContext(
+    private ScanState(
         String metalake,
         long policyId,
         Policy.BuiltInType policyType,
-        NearestMaintenancePolicyResolver nearest,
+        NearestMaintenancePolicyResolver nearestResolver,
         ExpandOccupancyGate occupancyGate,
         long minIntervalMs,
         long nowMs,
-        Set<Long> seenTables,
-        Map<Long, ExpandTableCandidate> enqueueable,
+        Set<Long> visitedTableIds,
+        Map<Long, ExpandTableCandidate> acceptedByTableId,
         int limit) {
       this.metalake = metalake;
       this.policyId = policyId;
       this.policyType = policyType;
-      this.nearest = nearest;
+      this.nearestResolver = nearestResolver;
       this.occupancyGate = occupancyGate;
       this.minIntervalMs = minIntervalMs;
       this.nowMs = nowMs;
-      this.seenTables = seenTables;
-      this.enqueueable = enqueueable;
+      this.visitedTableIds = visitedTableIds;
+      this.acceptedByTableId = acceptedByTableId;
       this.limit = limit;
     }
+
+    private boolean batchFull() {
+      return acceptedByTableId.size() >= limit;
+    }
   }
 }
diff --git 
a/plugins/maintenance/src/test/java/org/apache/gravitino/maintenance/scheduler/policy/expand/TestNearestPolicyTableEnumerator.java
 
b/plugins/maintenance/src/test/java/org/apache/gravitino/maintenance/scheduler/policy/expand/TestNearestPolicyTableEnumerator.java
index f2ef37d454..cbcc510013 100644
--- 
a/plugins/maintenance/src/test/java/org/apache/gravitino/maintenance/scheduler/policy/expand/TestNearestPolicyTableEnumerator.java
+++ 
b/plugins/maintenance/src/test/java/org/apache/gravitino/maintenance/scheduler/policy/expand/TestNearestPolicyTableEnumerator.java
@@ -21,6 +21,8 @@ package 
org.apache.gravitino.maintenance.scheduler.policy.expand;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 import java.time.Instant;
@@ -28,8 +30,12 @@ import java.util.List;
 import org.apache.gravitino.Entity;
 import org.apache.gravitino.MetadataObject;
 import org.apache.gravitino.MetadataObjects;
+import org.apache.gravitino.NameIdentifier;
 import org.apache.gravitino.RelationalEntity;
 import org.apache.gravitino.SupportsRelationOperations;
+import org.apache.gravitino.catalog.EntityCombinedTable;
+import org.apache.gravitino.catalog.SchemaDispatcher;
+import org.apache.gravitino.catalog.TableDispatcher;
 import org.apache.gravitino.meta.AuditInfo;
 import org.apache.gravitino.meta.GenericEntity;
 import org.apache.gravitino.meta.PolicyEntity;
@@ -39,6 +45,7 @@ import org.apache.gravitino.meta.TagEntity;
 import org.apache.gravitino.policy.IcebergDataCompactionContent;
 import org.apache.gravitino.policy.Policy;
 import org.apache.gravitino.policy.PolicyContents;
+import org.apache.gravitino.rel.Table;
 import org.apache.gravitino.storage.relational.service.PolicyTagRelService;
 import org.apache.gravitino.storage.relational.service.SchemaMetaService;
 import org.apache.gravitino.storage.relational.service.TableMetaService;
@@ -49,6 +56,7 @@ import org.apache.gravitino.utils.NamespaceUtil;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
 
 class TestNearestPolicyTableEnumerator {
 
@@ -59,6 +67,8 @@ class TestNearestPolicyTableEnumerator {
   private SchemaMetaService schemaMetaService;
   private TableMetaService tableMetaService;
   private NearestMaintenancePolicyResolver nearest;
+  private SchemaDispatcher schemaDispatcher;
+  private TableDispatcher tableDispatcher;
   private NearestPolicyTableEnumerator enumerator;
 
   @BeforeEach
@@ -68,13 +78,22 @@ class TestNearestPolicyTableEnumerator {
     schemaMetaService = mock(SchemaMetaService.class);
     tableMetaService = mock(TableMetaService.class);
     nearest = mock(NearestMaintenancePolicyResolver.class);
+    // Force meta fallback for tests that only stub SchemaMetaService / 
TableMetaService.
+    schemaDispatcher = mock(SchemaDispatcher.class);
+    tableDispatcher = mock(TableDispatcher.class);
+    when(schemaDispatcher.listSchemas(any()))
+        .thenThrow(new RuntimeException("use meta listing in unit test"));
+    when(tableDispatcher.listTables(any()))
+        .thenThrow(new RuntimeException("use meta listing in unit test"));
     enumerator =
         new NearestPolicyTableEnumerator(
             policyTagRelService,
             tagMetaService,
             schemaMetaService,
             tableMetaService,
-            () -> nearest);
+            () -> nearest,
+            schemaDispatcher,
+            tableDispatcher);
     
when(schemaMetaService.listSchemasByNamespace(any())).thenReturn(List.of());
   }
 
@@ -289,6 +308,107 @@ class TestNearestPolicyTableEnumerator {
         List.of(), tableIds(enumerator.listTableIds(policy, 10, allowAll(), 
0L)));
   }
 
+  @Test
+  void testImportsOnlyMissingTablesAfterCatalogList() throws Exception {
+    PolicyEntity policy = stubTaggedObject("catalog.schema", 
Entity.EntityType.SCHEMA);
+    SchemaDispatcher schemas = mock(SchemaDispatcher.class);
+    TableDispatcher tables = mock(TableDispatcher.class);
+    NameIdentifier known = NameIdentifierUtil.ofTable(METALAKE, "catalog", 
"schema", "known");
+    NameIdentifier missing = NameIdentifierUtil.ofTable(METALAKE, "catalog", 
"schema", "missing");
+    when(tables.listTables(eq(NamespaceUtil.ofTable(METALAKE, "catalog", 
"schema"))))
+        .thenReturn(new NameIdentifier[] {known, missing});
+    when(tableMetaService.listTablesByNamespace(
+            eq(NamespaceUtil.ofTable(METALAKE, "catalog", "schema"))))
+        .thenReturn(List.of(table(500L, "known")));
+    when(tables.loadTable(eq(missing)))
+        .thenReturn(
+            EntityCombinedTable.of(Mockito.mock(Table.class), table(501L, 
"missing"))
+                .withImported(true));
+
+    MetadataObject schemaObj =
+        MetadataObjects.of(
+            java.util.Arrays.asList("catalog", "schema"), 
MetadataObject.Type.SCHEMA);
+    stubWinner(tableMeta("catalog", "schema", "known"), schemaObj);
+    stubWinner(tableMeta("catalog", "schema", "missing"), schemaObj);
+
+    Assertions.assertEquals(
+        List.of(500L, 501L),
+        tableIds(catalogEnumerator(schemas, tables).listTableIds(policy, 10, 
allowAll(), 0L)));
+    verify(tables).loadTable(missing);
+    verify(tables, never()).loadTable(known);
+  }
+
+  @Test
+  void testImportsOnlyMissingSchemasAfterCatalogList() throws Exception {
+    PolicyEntity policy = stubTaggedObject("catalog", 
Entity.EntityType.CATALOG);
+    SchemaDispatcher schemas = mock(SchemaDispatcher.class);
+    TableDispatcher tables = mock(TableDispatcher.class);
+    NameIdentifier known = NameIdentifierUtil.ofSchema(METALAKE, "catalog", 
"known");
+    NameIdentifier missing = NameIdentifierUtil.ofSchema(METALAKE, "catalog", 
"missing");
+    when(schemas.listSchemas(eq(NamespaceUtil.ofSchema(METALAKE, "catalog"))))
+        .thenReturn(new NameIdentifier[] {known, missing});
+    
when(schemaMetaService.listSchemasByNamespace(eq(NamespaceUtil.ofSchema(METALAKE,
 "catalog"))))
+        .thenReturn(List.of(schema(2L, "known")));
+    
when(schemas.loadSchema(eq(missing))).thenReturn(mock(org.apache.gravitino.Schema.class));
+    when(tables.listTables(any())).thenReturn(new NameIdentifier[0]);
+    when(tableMetaService.listTablesByNamespace(any())).thenReturn(List.of());
+
+    Assertions.assertEquals(
+        List.of(),
+        tableIds(catalogEnumerator(schemas, tables).listTableIds(policy, 10, 
allowAll(), 0L)));
+    verify(schemas).loadSchema(missing);
+    verify(schemas, never()).loadSchema(known);
+  }
+
+  private PolicyEntity stubTaggedObject(String objectName, Entity.EntityType 
type)
+      throws Exception {
+    PolicyEntity policy = policy(10L, "compaction");
+    TagEntity tag = tag(1L, "maint");
+    when(policyTagRelService.listRelations(
+            eq(List.of(NameIdentifierUtil.ofPolicy(METALAKE, "compaction"))),
+            eq(Entity.EntityType.POLICY)))
+        .thenReturn(List.of(policyTagEdge(policy, tag)));
+    
when(tagMetaService.listAssociatedMetadataObjectsForTag(tag.nameIdentifier()))
+        .thenReturn(
+            List.of(
+                GenericEntity.builder()
+                    .withId(1L)
+                    .withName(objectName)
+                    .withEntityType(type)
+                    .build()));
+    return policy;
+  }
+
+  private void stubWinner(MetadataObject table, MetadataObject nearestObject) {
+    when(nearest.resolve(METALAKE, table, 
Policy.BuiltInType.ICEBERG_COMPACTION))
+        .thenReturn(NearestMaintenancePolicyResolver.Decision.winner(10L, 
nearestObject));
+  }
+
+  private NearestPolicyTableEnumerator catalogEnumerator(
+      SchemaDispatcher schemas, TableDispatcher tables) {
+    return new NearestPolicyTableEnumerator(
+        policyTagRelService,
+        tagMetaService,
+        schemaMetaService,
+        tableMetaService,
+        () -> nearest,
+        schemas,
+        tables);
+  }
+
+  private static MetadataObject tableMeta(String... parts) {
+    return MetadataObjects.of(java.util.Arrays.asList(parts), 
MetadataObject.Type.TABLE);
+  }
+
+  private static TableEntity table(long id, String name) {
+    return TableEntity.builder()
+        .withId(id)
+        .withName(name)
+        .withNamespace(NamespaceUtil.ofTable(METALAKE, "catalog", "schema"))
+        .withAuditInfo(audit())
+        .build();
+  }
+
   private static List<Long> tableIds(List<ExpandTableCandidate> candidates) {
     return candidates.stream().map(ExpandTableCandidate::tableId).toList();
   }

Reply via email to