This is an automated email from the ASF dual-hosted git repository.
yuqi4733 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 2b31bb2eb9 [#10459] improvement(core): Eliminate N+1 queries in
metadata object (#10589)
2b31bb2eb9 is described below
commit 2b31bb2eb9a78e3cd82ca79019a8423886d7aafa
Author: Babu Mahesh <[email protected]>
AuthorDate: Thu Apr 2 08:44:27 2026 +0530
[#10459] improvement(core): Eliminate N+1 queries in metadata object
(#10589)
### What changes were proposed in this pull request?
Eliminates N+1 query problems when resolving metadata object names for
Tags, Policies, and Job Templates by implementing batch query methods.
### Why are the changes needed
- Previously, when fetching names for multiple metadata objects
(tags/policies/job templates), the code would issue one SQL query per
object ID in a stream. This creates N+1 query performance problems,
especially when dealing with large numbers of objects.
- Implemented batch query methods using MyBatis `<foreach>` to fetch
multiple records in a single SQL query with an `IN` clause.
- The service layer now batches all IDs and makes one query instead of N
individual queries.
Fix: #10459
### How was this patch tested?
All the existing unit tests and integrations tests ran and should be
sufficient as there is no behaviour changes
---------
Co-authored-by: Babu Mahesh <[email protected]>
---
.../relational/mapper/JobTemplateMetaMapper.java | 6 +
.../mapper/JobTemplateMetaSQLProviderFactory.java | 5 +
.../relational/mapper/PolicyMetaMapper.java | 13 +
.../mapper/PolicyMetaSQLProviderFactory.java | 4 +
.../storage/relational/mapper/TagMetaMapper.java | 3 +
.../mapper/TagMetaSQLProviderFactory.java | 4 +
.../base/JobTemplateMetaBaseSQLProvider.java | 20 ++
.../provider/base/PolicyMetaBaseSQLProvider.java | 17 ++
.../provider/base/TagMetaBaseSQLProvider.java | 21 ++
.../relational/service/MetadataObjectService.java | 87 ++++---
.../storage/relational/TestJDBCBackend.java | 96 +++++++
.../service/TestJobTemplateMetaService.java | 37 ---
.../service/TestMetadataObjectService.java | 285 +++++++++++++++++++++
13 files changed, 530 insertions(+), 68 deletions(-)
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/JobTemplateMetaMapper.java
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/JobTemplateMetaMapper.java
index c264294e32..40e1100520 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/JobTemplateMetaMapper.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/JobTemplateMetaMapper.java
@@ -80,6 +80,12 @@ public interface JobTemplateMetaMapper {
@SelectProvider(type = JobTemplateMetaSQLProviderFactory.class, method =
"selectJobTemplateById")
JobTemplatePO selectJobTemplateById(Long jobTemplateId);
+ @SelectProvider(
+ type = JobTemplateMetaSQLProviderFactory.class,
+ method = "listJobTemplatePOsByJobTemplateIds")
+ List<JobTemplatePO> listJobTemplatePOsByJobTemplateIds(
+ @Param("jobTemplateIds") List<Long> jobTemplateIds);
+
@SelectProvider(
type = JobTemplateMetaSQLProviderFactory.class,
method = "batchSelectJobTemplateByIdentifier")
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/JobTemplateMetaSQLProviderFactory.java
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/JobTemplateMetaSQLProviderFactory.java
index 0598a9a9a0..04383da401 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/JobTemplateMetaSQLProviderFactory.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/JobTemplateMetaSQLProviderFactory.java
@@ -104,6 +104,11 @@ public class JobTemplateMetaSQLProviderFactory {
return getProvider().selectJobTemplateById(jobTemplateId);
}
+ public static String listJobTemplatePOsByJobTemplateIds(
+ @Param("jobTemplateIds") List<Long> jobTemplateIds) {
+ return getProvider().listJobTemplatePOsByJobTemplateIds(jobTemplateIds);
+ }
+
public static String batchSelectJobTemplateByIdentifier(
@Param("metalakeName") String metalakeName,
@Param("jobTemplateNames") List<String> jobTemplateNames) {
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/PolicyMetaMapper.java
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/PolicyMetaMapper.java
index e2e35cc6e0..9862c5f151 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/PolicyMetaMapper.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/PolicyMetaMapper.java
@@ -159,6 +159,19 @@ public interface PolicyMetaMapper {
@SelectProvider(type = PolicyMetaSQLProviderFactory.class, method =
"selectPolicyByPolicyId")
PolicyPO selectPolicyByPolicyId(@Param("policyId") Long policyId);
+ @Results({
+ @Result(property = "policyId", column = "policy_id"),
+ @Result(property = "policyName", column = "policy_name"),
+ @Result(property = "policyType", column = "policy_type"),
+ @Result(property = "metalakeId", column = "metalake_id"),
+ @Result(property = "auditInfo", column = "audit_info"),
+ @Result(property = "currentVersion", column = "current_version"),
+ @Result(property = "lastVersion", column = "last_version"),
+ @Result(property = "deletedAt", column = "deleted_at")
+ })
+ @SelectProvider(type = PolicyMetaSQLProviderFactory.class, method =
"listPolicyPOsByPolicyIds")
+ List<PolicyPO> listPolicyPOsByPolicyIds(@Param("policyIds") List<Long>
policyIds);
+
@Results({
@Result(property = "policyId", column = "policy_id"),
@Result(property = "policyName", column = "policy_name"),
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/PolicyMetaSQLProviderFactory.java
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/PolicyMetaSQLProviderFactory.java
index afb257227a..4f7b1d1177 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/PolicyMetaSQLProviderFactory.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/PolicyMetaSQLProviderFactory.java
@@ -100,6 +100,10 @@ public class PolicyMetaSQLProviderFactory {
return getProvider().selectPolicyByPolicyId(policyId);
}
+ public static String listPolicyPOsByPolicyIds(@Param("policyIds") List<Long>
policyIds) {
+ return getProvider().listPolicyPOsByPolicyIds(policyIds);
+ }
+
public static String batchSelectPolicyByIdentifier(
@Param("metalakeName") String metalakeName, @Param("policyNames")
List<String> policyNames) {
return getProvider().batchSelectPolicyByIdentifier(metalakeName,
policyNames);
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaMapper.java
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaMapper.java
index 7732c8b1a0..68cb3d721f 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaMapper.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaMapper.java
@@ -80,6 +80,9 @@ public interface TagMetaMapper {
@SelectProvider(type = TagMetaSQLProviderFactory.class, method =
"selectTagByTagId")
TagPO selectTagByTagId(@Param("tagId") Long tagId);
+ @SelectProvider(type = TagMetaSQLProviderFactory.class, method =
"listTagPOsByTagIds")
+ List<TagPO> listTagPOsByTagIds(@Param("tagIds") List<Long> tagIds);
+
@SelectProvider(type = TagMetaSQLProviderFactory.class, method =
"batchSelectTagByIdentifier")
List<TagPO> batchSelectTagByIdentifier(
@Param("metalakeName") String metalakeName, @Param("tagNames")
List<String> tagNames);
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaSQLProviderFactory.java
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaSQLProviderFactory.java
index 7273279e6b..1dde368e69 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaSQLProviderFactory.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaSQLProviderFactory.java
@@ -106,6 +106,10 @@ public class TagMetaSQLProviderFactory {
return getProvider().selectTagByTagId(tagId);
}
+ public static String listTagPOsByTagIds(@Param("tagIds") List<Long> tagIds) {
+ return getProvider().listTagPOsByTagIds(tagIds);
+ }
+
public static String batchSelectTagByIdentifier(
@Param("metalakeName") String metalakeName, @Param("tagNames")
List<String> tagNames) {
return getProvider().batchSelectTagByIdentifier(metalakeName, tagNames);
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/JobTemplateMetaBaseSQLProvider.java
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/JobTemplateMetaBaseSQLProvider.java
index 782dd362a8..807eba1cdc 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/JobTemplateMetaBaseSQLProvider.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/JobTemplateMetaBaseSQLProvider.java
@@ -163,6 +163,26 @@ public class JobTemplateMetaBaseSQLProvider {
+ " AND jtm.deleted_at = 0";
}
+ public String listJobTemplatePOsByJobTemplateIds(
+ @Param("jobTemplateIds") List<Long> jobTemplateIds) {
+ return "<script>"
+ + "SELECT jtm.job_template_id AS jobTemplateId, jtm.job_template_name
AS jobTemplateName,"
+ + " jtm.metalake_id AS metalakeId, jtm.job_template_comment AS
jobTemplateComment,"
+ + " jtm.job_template_content AS jobTemplateContent, jtm.audit_info AS
auditInfo,"
+ + " jtm.current_version AS currentVersion, jtm.last_version AS
lastVersion,"
+ + " jtm.deleted_at AS deletedAt"
+ + " FROM "
+ + JobTemplateMetaMapper.TABLE_NAME
+ + " jtm"
+ + " WHERE jtm.deleted_at = 0"
+ + " AND jtm.job_template_id IN ("
+ + "<foreach collection='jobTemplateIds' item='jobTemplateId'
separator=','>"
+ + "#{jobTemplateId}"
+ + "</foreach>"
+ + ")"
+ + "</script>";
+ }
+
public String batchSelectJobTemplateByIdentifier(
@Param("metalakeName") String metalakeName,
@Param("jobTemplateNames") List<String> jobTemplateNames) {
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/PolicyMetaBaseSQLProvider.java
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/PolicyMetaBaseSQLProvider.java
index fa25c66aec..c0f6b21f6d 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/PolicyMetaBaseSQLProvider.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/PolicyMetaBaseSQLProvider.java
@@ -184,6 +184,23 @@ public class PolicyMetaBaseSQLProvider {
+ " AND pm.deleted_at = 0 ";
}
+ public String listPolicyPOsByPolicyIds(@Param("policyIds") List<Long>
policyIds) {
+ return "<script>"
+ + "SELECT pm.policy_id, pm.policy_name, pm.policy_type,
pm.metalake_id,"
+ + " pm.audit_info, pm.current_version, pm.last_version,"
+ + " pm.deleted_at"
+ + " FROM "
+ + POLICY_META_TABLE_NAME
+ + " pm"
+ + " WHERE pm.deleted_at = 0"
+ + " AND pm.policy_id IN ("
+ + "<foreach collection='policyIds' item='policyId' separator=','>"
+ + "#{policyId}"
+ + "</foreach>"
+ + ")"
+ + "</script>";
+ }
+
public String selectPolicyMetaByMetalakeIdAndName(
@Param("metalakeId") Long metalakeId, @Param("policyName") String
policyName) {
return "SELECT pm.policy_id, pm.policy_name, pm.policy_type,
pm.metalake_id,"
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/TagMetaBaseSQLProvider.java
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/TagMetaBaseSQLProvider.java
index 74dbfaeeb7..8aac66bf88 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/TagMetaBaseSQLProvider.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/TagMetaBaseSQLProvider.java
@@ -226,6 +226,27 @@ public class TagMetaBaseSQLProvider {
+ " WHERE tag_id = #{tagId} and deleted_at = 0";
}
+ public String listTagPOsByTagIds(@Param("tagIds") List<Long> tagIds) {
+ return "<script>"
+ + "SELECT tag_id as tagId, tag_name as tagName,"
+ + " metalake_id as metalakeId,"
+ + " tag_comment as comment,"
+ + " properties as properties,"
+ + " audit_info as auditInfo,"
+ + " current_version as currentVersion,"
+ + " last_version as lastVersion,"
+ + " deleted_at as deletedAt"
+ + " FROM "
+ + TAG_TABLE_NAME
+ + " WHERE deleted_at = 0"
+ + " AND tag_id IN ("
+ + "<foreach collection='tagIds' item='tagId' separator=','>"
+ + "#{tagId}"
+ + "</foreach>"
+ + ")"
+ + "</script>";
+ }
+
public String batchSelectTagByIdentifier(
@Param("metalakeName") String metalakeName, @Param("tagNames")
List<String> tagNames) {
return "<script>"
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java
b/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java
index 06656ba964..efb7c7e8d3 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java
@@ -50,10 +50,13 @@ import
org.apache.gravitino.storage.relational.mapper.ViewMetaMapper;
import org.apache.gravitino.storage.relational.po.CatalogPO;
import org.apache.gravitino.storage.relational.po.ColumnPO;
import org.apache.gravitino.storage.relational.po.FilesetPO;
+import org.apache.gravitino.storage.relational.po.JobTemplatePO;
import org.apache.gravitino.storage.relational.po.MetalakePO;
import org.apache.gravitino.storage.relational.po.ModelPO;
+import org.apache.gravitino.storage.relational.po.PolicyPO;
import org.apache.gravitino.storage.relational.po.SchemaPO;
import org.apache.gravitino.storage.relational.po.TablePO;
+import org.apache.gravitino.storage.relational.po.TagPO;
import org.apache.gravitino.storage.relational.po.TopicPO;
import org.apache.gravitino.storage.relational.po.ViewPO;
import org.apache.gravitino.storage.relational.utils.SessionUtils;
@@ -92,17 +95,23 @@ public class MetadataObjectService {
private static Map<Long, String> getPolicyObjectsFullName(List<Long>
policyIds) {
if (policyIds == null || policyIds.isEmpty()) {
- return Map.of();
+ return Maps.newHashMap();
+ }
+
+ List<PolicyPO> policyPOs =
+ SessionUtils.getWithoutCommit(
+ PolicyMetaMapper.class, mapper ->
mapper.listPolicyPOsByPolicyIds(policyIds));
+
+ if (policyPOs == null || policyPOs.isEmpty()) {
+ return Maps.newHashMap();
}
- return policyIds.stream()
- .collect(
- Collectors.toMap(
- policyId -> policyId,
- policyId ->
- SessionUtils.getWithoutCommit(
- PolicyMetaMapper.class,
- policyMetaMapper ->
-
policyMetaMapper.selectPolicyByPolicyId(policyId).getPolicyName())));
+
+ HashMap<Long, String> policyIdAndNameMap = new HashMap<>();
+
+ policyPOs.forEach(
+ policyPO -> policyIdAndNameMap.put(policyPO.getPolicyId(),
policyPO.getPolicyName()));
+
+ return policyIdAndNameMap;
}
private static Map<Long, String> getJobObjectsFullName(List<Long> jobIds) {
@@ -119,31 +128,43 @@ public class MetadataObjectService {
return Maps.newHashMap();
}
- return jobTemplateIds.stream()
- .collect(
- Collectors.toMap(
- jobTemplateId -> jobTemplateId,
- jobTemplateId ->
- SessionUtils.getWithoutCommit(
- JobTemplateMetaMapper.class,
- jobTemplateMetaMapper ->
- jobTemplateMetaMapper
- .selectJobTemplateById(jobTemplateId)
- .jobTemplateName())));
+ List<JobTemplatePO> jobTemplatePOs =
+ SessionUtils.getWithoutCommit(
+ JobTemplateMetaMapper.class,
+ mapper ->
mapper.listJobTemplatePOsByJobTemplateIds(jobTemplateIds));
+
+ if (jobTemplatePOs == null || jobTemplatePOs.isEmpty()) {
+ return Maps.newHashMap();
+ }
+
+ HashMap<Long, String> jobTemplateIdAndNameMap = new HashMap<>();
+
+ jobTemplatePOs.forEach(
+ jobTemplatePO ->
+ jobTemplateIdAndNameMap.put(
+ jobTemplatePO.jobTemplateId(),
jobTemplatePO.jobTemplateName()));
+
+ return jobTemplateIdAndNameMap;
}
private static Map<Long, String> getTagObjectsFullName(List<Long> tagIds) {
if (tagIds == null || tagIds.isEmpty()) {
- return Map.of();
+ return Maps.newHashMap();
+ }
+
+ List<TagPO> tagPOs =
+ SessionUtils.getWithoutCommit(
+ TagMetaMapper.class, mapper -> mapper.listTagPOsByTagIds(tagIds));
+
+ if (tagPOs == null || tagPOs.isEmpty()) {
+ return Maps.newHashMap();
}
- return tagIds.stream()
- .collect(
- Collectors.toMap(
- tagId -> tagId,
- tagId ->
- SessionUtils.getWithoutCommit(
- TagMetaMapper.class,
- tagMetaMapper ->
tagMetaMapper.selectTagByTagId(tagId).getTagName())));
+
+ HashMap<Long, String> tagIdAndNameMap = new HashMap<>();
+
+ tagPOs.forEach(tagPO -> tagIdAndNameMap.put(tagPO.getTagId(),
tagPO.getTagName()));
+
+ return tagIdAndNameMap;
}
private MetadataObjectService() {}
@@ -317,12 +338,16 @@ public class MetadataObjectService {
metricsSource = GRAVITINO_RELATIONAL_STORE_METRIC_NAME,
baseMetricName = "getTableObjectsFullName")
public static Map<Long, String> getTableObjectsFullName(List<Long> tableIds)
{
+ if (tableIds == null || tableIds.isEmpty()) {
+ return Maps.newHashMap();
+ }
+
List<TablePO> tablePOs =
SessionUtils.getWithoutCommit(
TableMetaMapper.class, mapper ->
mapper.listTablePOsByTableIds(tableIds));
if (tablePOs == null || tablePOs.isEmpty()) {
- return new HashMap<>();
+ return Maps.newHashMap();
}
List<Long> schemaIds =
tablePOs.stream().map(TablePO::getSchemaId).collect(Collectors.toList());
diff --git
a/core/src/test/java/org/apache/gravitino/storage/relational/TestJDBCBackend.java
b/core/src/test/java/org/apache/gravitino/storage/relational/TestJDBCBackend.java
index c34ec55491..00a9a5aed4 100644
---
a/core/src/test/java/org/apache/gravitino/storage/relational/TestJDBCBackend.java
+++
b/core/src/test/java/org/apache/gravitino/storage/relational/TestJDBCBackend.java
@@ -43,24 +43,30 @@ import org.apache.gravitino.authorization.SecurableObjects;
import org.apache.gravitino.file.Fileset;
import org.apache.gravitino.integration.test.util.CloseContainerExtension;
import org.apache.gravitino.integration.test.util.PrintFuncNameExtension;
+import org.apache.gravitino.job.ShellJobTemplate;
+import org.apache.gravitino.job.SparkJobTemplate;
import org.apache.gravitino.meta.AuditInfo;
import org.apache.gravitino.meta.BaseMetalake;
import org.apache.gravitino.meta.CatalogEntity;
import org.apache.gravitino.meta.FilesetEntity;
import org.apache.gravitino.meta.GenericEntity;
import org.apache.gravitino.meta.GroupEntity;
+import org.apache.gravitino.meta.JobTemplateEntity;
import org.apache.gravitino.meta.ModelEntity;
import org.apache.gravitino.meta.PolicyEntity;
import org.apache.gravitino.meta.RoleEntity;
import org.apache.gravitino.meta.SchemaEntity;
import org.apache.gravitino.meta.SchemaVersion;
import org.apache.gravitino.meta.TableEntity;
+import org.apache.gravitino.meta.TagEntity;
import org.apache.gravitino.meta.TopicEntity;
import org.apache.gravitino.meta.UserEntity;
import org.apache.gravitino.policy.Policy;
+import org.apache.gravitino.policy.PolicyContent;
import org.apache.gravitino.policy.PolicyContents;
import org.apache.gravitino.storage.RandomIdGenerator;
import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper;
+import org.apache.gravitino.utils.NamespaceUtil;
import org.apache.ibatis.session.SqlSession;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.TestInstance;
@@ -402,6 +408,14 @@ public abstract class TestJDBCBackend {
.build();
}
+ protected TableEntity createAndInsertTableEntity(Namespace namespace, String
name)
+ throws IOException {
+ TableEntity tableEntity =
+ createTableEntity(RandomIdGenerator.INSTANCE.nextId(), namespace,
name, AUDIT_INFO);
+ backend.insert(tableEntity, false);
+ return tableEntity;
+ }
+
protected FilesetEntity createFilesetEntity(
Long id, Namespace namespace, String name, AuditInfo auditInfo) {
return FilesetEntity.builder()
@@ -541,4 +555,86 @@ public abstract class TestJDBCBackend {
createAndInsertCatalog(metalakeName, catalogName);
createAndInsertSchema(metalakeName, catalogName, schemaName);
}
+
+ protected static JobTemplateEntity newShellJobTemplateEntity(
+ String name, String comment, String metalake) {
+ ShellJobTemplate shellJobTemplate =
+ ShellJobTemplate.builder()
+ .withName(name)
+ .withComment(comment)
+ .withExecutable("/bin/echo")
+ .build();
+
+ return JobTemplateEntity.builder()
+ .withId(RandomIdGenerator.INSTANCE.nextId())
+ .withName(name)
+ .withNamespace(NamespaceUtil.ofJobTemplate(metalake))
+
.withTemplateContent(JobTemplateEntity.TemplateContent.fromJobTemplate(shellJobTemplate))
+ .withAuditInfo(AUDIT_INFO)
+ .build();
+ }
+
+ protected static JobTemplateEntity newSparkJobTemplateEntity(
+ String name, String comment, String metalake) {
+ SparkJobTemplate sparkJobTemplate =
+ SparkJobTemplate.builder()
+ .withName(name)
+ .withComment(comment)
+ .withClassName("org.apache.spark.examples.SparkPi")
+ .withExecutable("file:/path/to/spark-examples.jar")
+ .build();
+
+ return JobTemplateEntity.builder()
+ .withId(RandomIdGenerator.INSTANCE.nextId())
+ .withName(name)
+ .withNamespace(NamespaceUtil.ofJobTemplate(metalake))
+
.withTemplateContent(JobTemplateEntity.TemplateContent.fromJobTemplate(sparkJobTemplate))
+ .withAuditInfo(AUDIT_INFO)
+ .build();
+ }
+
+ protected JobTemplateEntity createAndInsertShellJobTemplateEntity(
+ String name, String comment, String metalake) throws IOException {
+ JobTemplateEntity jobTemplateEntity = newShellJobTemplateEntity(name,
comment, metalake);
+ backend.insert(jobTemplateEntity, false);
+ return jobTemplateEntity;
+ }
+
+ protected JobTemplateEntity createAndInsertSparkJobTemplateEntity(
+ String name, String comment, String metalake) throws IOException {
+ JobTemplateEntity jobTemplateEntity = newSparkJobTemplateEntity(name,
comment, metalake);
+ backend.insert(jobTemplateEntity, false);
+ return jobTemplateEntity;
+ }
+
+ protected TagEntity createAndInsertTagEntity(String name, String comment,
String metalake)
+ throws IOException {
+ TagEntity tagEntity =
+ TagEntity.builder()
+ .withId(RandomIdGenerator.INSTANCE.nextId())
+ .withName(name)
+ .withNamespace(NamespaceUtil.ofTag(metalake))
+ .withComment(comment)
+ .withAuditInfo(AUDIT_INFO)
+ .build();
+ backend.insert(tagEntity, false);
+ return tagEntity;
+ }
+
+ protected PolicyEntity createAndInsertPolicyEntity(
+ String policyName, String comment, PolicyContent policyContent, String
metalake)
+ throws IOException {
+ PolicyEntity policyEntity =
+ PolicyEntity.builder()
+ .withId(RandomIdGenerator.INSTANCE.nextId())
+ .withName(policyName)
+ .withNamespace(NamespaceUtil.ofPolicy(metalake))
+ .withComment(comment)
+ .withPolicyType(Policy.BuiltInType.CUSTOM)
+ .withContent(policyContent)
+ .withAuditInfo(AUDIT_INFO)
+ .build();
+ backend.insert(policyEntity, false);
+ return policyEntity;
+ }
}
diff --git
a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestJobTemplateMetaService.java
b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestJobTemplateMetaService.java
index 76ca0203a1..3dfe80f408 100644
---
a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestJobTemplateMetaService.java
+++
b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestJobTemplateMetaService.java
@@ -24,8 +24,6 @@ import java.util.List;
import org.apache.gravitino.EntityAlreadyExistsException;
import org.apache.gravitino.exceptions.NoSuchEntityException;
import org.apache.gravitino.job.JobHandle;
-import org.apache.gravitino.job.ShellJobTemplate;
-import org.apache.gravitino.job.SparkJobTemplate;
import org.apache.gravitino.meta.AuditInfo;
import org.apache.gravitino.meta.BaseMetalake;
import org.apache.gravitino.meta.JobEntity;
@@ -278,41 +276,6 @@ public class TestJobTemplateMetaService extends
TestJDBCBackend {
updatedJobTemplateEntity.nameIdentifier(), e ->
duplicateNameJobTemplateEntity));
}
- static JobTemplateEntity newShellJobTemplateEntity(String name, String
comment, String metalake) {
- ShellJobTemplate shellJobTemplate =
- ShellJobTemplate.builder()
- .withName(name)
- .withComment(comment)
- .withExecutable("/bin/echo")
- .build();
-
- return JobTemplateEntity.builder()
- .withId(RandomIdGenerator.INSTANCE.nextId())
- .withName(name)
- .withNamespace(NamespaceUtil.ofJobTemplate(metalake))
-
.withTemplateContent(JobTemplateEntity.TemplateContent.fromJobTemplate(shellJobTemplate))
- .withAuditInfo(AUDIT_INFO)
- .build();
- }
-
- static JobTemplateEntity newSparkJobTemplateEntity(String name, String
comment, String metalake) {
- SparkJobTemplate sparkJobTemplate =
- SparkJobTemplate.builder()
- .withName(name)
- .withComment(comment)
- .withClassName("org.apache.spark.examples.SparkPi")
- .withExecutable("file:/path/to/spark-examples.jar")
- .build();
-
- return JobTemplateEntity.builder()
- .withId(RandomIdGenerator.INSTANCE.nextId())
- .withName(name)
- .withNamespace(NamespaceUtil.ofJobTemplate(metalake))
-
.withTemplateContent(JobTemplateEntity.TemplateContent.fromJobTemplate(sparkJobTemplate))
- .withAuditInfo(AUDIT_INFO)
- .build();
- }
-
static JobEntity newJobEntity(String templateName, JobHandle.Status status,
String metalake) {
return JobEntity.builder()
.withId(RandomIdGenerator.INSTANCE.nextId())
diff --git
a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestMetadataObjectService.java
b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestMetadataObjectService.java
new file mode 100644
index 0000000000..4d14e21b5c
--- /dev/null
+++
b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestMetadataObjectService.java
@@ -0,0 +1,285 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.storage.relational.service;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.apache.gravitino.Entity;
+import org.apache.gravitino.MetadataObject;
+import org.apache.gravitino.Namespace;
+import org.apache.gravitino.meta.CatalogEntity;
+import org.apache.gravitino.meta.GenericEntity;
+import org.apache.gravitino.meta.JobTemplateEntity;
+import org.apache.gravitino.meta.PolicyEntity;
+import org.apache.gravitino.meta.TableEntity;
+import org.apache.gravitino.meta.TagEntity;
+import org.apache.gravitino.policy.PolicyContent;
+import org.apache.gravitino.policy.PolicyContents;
+import org.apache.gravitino.storage.relational.TestJDBCBackend;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.TestTemplate;
+
+public class TestMetadataObjectService extends TestJDBCBackend {
+ private static final String METALAKE_NAME =
"metalake_for_metadata_object_test";
+
+ private final Set<MetadataObject.Type> supportedObjectTypes =
+ ImmutableSet.of(
+ MetadataObject.Type.CATALOG,
+ MetadataObject.Type.SCHEMA,
+ MetadataObject.Type.TABLE,
+ MetadataObject.Type.FILESET,
+ MetadataObject.Type.MODEL,
+ MetadataObject.Type.TOPIC);
+ private final PolicyContent policyContent =
+ PolicyContents.custom(ImmutableMap.of("field", 123),
supportedObjectTypes, null);
+
+ @BeforeEach
+ public void prepare() throws IOException {
+ createAndInsertMakeLake(METALAKE_NAME);
+ }
+
+ @TestTemplate
+ public void testBatchGetPolicyObjectsFullName() throws IOException {
+
+ // Create multiple policies
+ String policyOne = "policy_one";
+ String policyTwo = "policy_two";
+ PolicyEntity policy1 =
+ createAndInsertPolicyEntity(policyOne, "test policy 1", policyContent,
METALAKE_NAME);
+ createAndInsertPolicyEntity(policyTwo, "test policy 2", policyContent,
METALAKE_NAME);
+
+ // Batch query using MetadataObjectService to fetch only one Entity
belongs to policy1
+ List<GenericEntity> genericEntities =
+ List.of(
+ GenericEntity.builder()
+ .withId(policy1.id())
+ .withName(policy1.name())
+ .withNamespace(policy1.namespace())
+ .withEntityType(Entity.EntityType.POLICY)
+ .build());
+
+ // Call MetadataObjectService.fromGenericEntities which will internally
call
+ // listPolicyPOsByPolicyIds
+ List<MetadataObject> metadataObjects =
+ MetadataObjectService.fromGenericEntities(genericEntities);
+
+ // Verify results
+ assertEquals(1, metadataObjects.size());
+ Set<String> policyNames =
+
metadataObjects.stream().map(MetadataObject::name).collect(Collectors.toSet());
+ assertTrue(policyNames.contains(policyOne));
+
+ // Verify all are POLICY type
+ assertTrue(metadataObjects.stream().allMatch(obj -> obj.type() ==
MetadataObject.Type.POLICY));
+ }
+
+ @TestTemplate
+ public void testBatchGetMixedObjectsFullName() throws IOException {
+ String policyOne = "policy_one";
+ String policyTwo = "policy_two";
+ String catalogOne = "catalog_one";
+ PolicyEntity policy1 =
+ createAndInsertPolicyEntity(policyOne, "test policy 1", policyContent,
METALAKE_NAME);
+ createAndInsertPolicyEntity(policyTwo, "test policy 2", policyContent,
METALAKE_NAME);
+ CatalogEntity catalog = createAndInsertCatalog(METALAKE_NAME, catalogOne);
+
+ List<GenericEntity> mixedEntities =
+ List.of(
+ GenericEntity.builder()
+ .withId(policy1.id())
+ .withName(policy1.name())
+ .withNamespace(policy1.namespace())
+ .withEntityType(Entity.EntityType.POLICY)
+ .build(),
+ GenericEntity.builder()
+ .withId(catalog.id())
+ .withName(catalog.name())
+ .withNamespace(catalog.namespace())
+ .withEntityType(Entity.EntityType.CATALOG)
+ .build());
+
+ List<MetadataObject> mixedResult =
MetadataObjectService.fromGenericEntities(mixedEntities);
+ assertEquals(2, mixedResult.size());
+
+ Map<MetadataObject.Type, Long> typeCounts =
+ mixedResult.stream()
+ .collect(Collectors.groupingBy(MetadataObject::type,
Collectors.counting()));
+ assertEquals(1L, typeCounts.get(MetadataObject.Type.POLICY));
+ assertEquals(1L, typeCounts.get(MetadataObject.Type.CATALOG));
+
+ // Verify names match
+ MetadataObject policyObject =
+ mixedResult.stream()
+ .filter(obj -> obj.type() == MetadataObject.Type.POLICY)
+ .findFirst()
+ .orElseThrow();
+ assertEquals(policyOne, policyObject.name());
+
+ MetadataObject catalogObject =
+ mixedResult.stream()
+ .filter(obj -> obj.type() == MetadataObject.Type.CATALOG)
+ .findFirst()
+ .orElseThrow();
+ assertEquals(catalogOne, catalogObject.name());
+ }
+
+ @TestTemplate
+ public void testBatchGetMetadataObjectsFullNameWithEmptyList() {
+ // Test with empty list - should return empty result
+ List<MetadataObject> emptyResult =
MetadataObjectService.fromGenericEntities(List.of());
+ assertEquals(0, emptyResult.size());
+ }
+
+ @TestTemplate
+ public void testBatchGetTagObjectsFullName() throws IOException {
+
+ // Create multiple tags
+ String tagOne = "tag_one";
+ String tagTwo = "tag_two";
+ TagEntity tag1 = createAndInsertTagEntity(tagOne, "test tag 1",
METALAKE_NAME);
+ TagEntity tag2 = createAndInsertTagEntity(tagTwo, "test tag 2",
METALAKE_NAME);
+ // Test batch query using MetadataObjectService
+ List<GenericEntity> genericEntities =
+ List.of(
+ GenericEntity.builder()
+ .withId(tag1.id())
+ .withName(tag1.name())
+ .withNamespace(tag1.namespace())
+ .withEntityType(Entity.EntityType.TAG)
+ .build(),
+ GenericEntity.builder()
+ .withId(tag2.id())
+ .withName(tag2.name())
+ .withNamespace(tag2.namespace())
+ .withEntityType(Entity.EntityType.TAG)
+ .build());
+
+ List<MetadataObject> metadataObjects =
+ MetadataObjectService.fromGenericEntities(genericEntities);
+
+ // Verify results
+ assertEquals(2, metadataObjects.size());
+ Set<String> tagNames =
+
metadataObjects.stream().map(MetadataObject::name).collect(Collectors.toSet());
+ Set<String> expected = Set.of(tagOne, tagTwo);
+ assertEquals(expected, tagNames);
+
+ // Verify all are TAG type
+ assertTrue(metadataObjects.stream().allMatch(obj -> obj.type() ==
MetadataObject.Type.TAG));
+ }
+
+ @TestTemplate
+ public void testBatchGetJobTemplateObjectsFullName() throws IOException {
+ // Create multiple job templates
+ String shellTemplate = "shell_template";
+ String sparkTemplate = "spark_template";
+ JobTemplateEntity shellJObTemplate =
+ createAndInsertShellJobTemplateEntity(shellTemplate, "A shell job
template", METALAKE_NAME);
+ JobTemplateEntity sparkJobTemplate =
+ createAndInsertSparkJobTemplateEntity(sparkTemplate, "A spark job
template", METALAKE_NAME);
+
+ // Test batch query using MetadataObjectService
+ List<GenericEntity> jobTemplateEntities =
+ List.of(
+ GenericEntity.builder()
+ .withId(shellJObTemplate.id())
+ .withName(shellJObTemplate.name())
+ .withNamespace(shellJObTemplate.namespace())
+ .withEntityType(Entity.EntityType.JOB_TEMPLATE)
+ .build(),
+ GenericEntity.builder()
+ .withId(sparkJobTemplate.id())
+ .withName(sparkJobTemplate.name())
+ .withNamespace(sparkJobTemplate.namespace())
+ .withEntityType(Entity.EntityType.JOB_TEMPLATE)
+ .build());
+
+ List<MetadataObject> metadataObjects =
+ MetadataObjectService.fromGenericEntities(jobTemplateEntities);
+
+ // Verify results
+ assertEquals(2, metadataObjects.size());
+ Set<String> jobTemplateNames =
+
metadataObjects.stream().map(MetadataObject::name).collect(Collectors.toSet());
+
+ Set<String> expected = Set.of(shellTemplate, sparkTemplate);
+ assertEquals(expected, jobTemplateNames);
+
+ // Verify all are JOB_TEMPLATE type
+ assertTrue(
+ metadataObjects.stream().allMatch(obj -> obj.type() ==
MetadataObject.Type.JOB_TEMPLATE));
+ }
+
+ @TestTemplate
+ public void testBatchGetTableObjectsFullName() throws IOException {
+ // Create catalog and schema
+ String catalogName = "test_catalog";
+ String schemaName = "test_schema";
+ String testTableOne = "test_table_one";
+ String testTableTwo = "test_table_two";
+ Namespace namespace = Namespace.of(METALAKE_NAME, catalogName, schemaName);
+ createAndInsertCatalog(METALAKE_NAME, catalogName);
+ createAndInsertSchema(METALAKE_NAME, catalogName, schemaName);
+ // Create multiple tables
+ TableEntity table1 = createAndInsertTableEntity(namespace, testTableOne);
+ TableEntity table2 = createAndInsertTableEntity(namespace, testTableTwo);
+
+ // Test batch query using MetadataObjectService
+ List<GenericEntity> tableEntities =
+ List.of(
+ GenericEntity.builder()
+ .withId(table1.id())
+ .withName(table1.name())
+ .withNamespace(table1.namespace())
+ .withEntityType(Entity.EntityType.TABLE)
+ .build(),
+ GenericEntity.builder()
+ .withId(table2.id())
+ .withName(table2.name())
+ .withNamespace(table2.namespace())
+ .withEntityType(Entity.EntityType.TABLE)
+ .build());
+
+ List<MetadataObject> metadataObjects =
MetadataObjectService.fromGenericEntities(tableEntities);
+
+ // Verify results
+ assertEquals(2, metadataObjects.size());
+
+ // Table full names include catalog.schema.table format
+ Set<String> tableFullNames =
+
metadataObjects.stream().map(MetadataObject::fullName).collect(Collectors.toSet());
+
+ Set<String> expected =
+ Set.of(
+ catalogName + "." + schemaName + "." + testTableOne,
+ catalogName + "." + schemaName + "." + testTableTwo);
+ assertEquals(expected, tableFullNames);
+
+ // Verify all are TABLE type
+ assertTrue(metadataObjects.stream().allMatch(obj -> obj.type() ==
MetadataObject.Type.TABLE));
+ }
+}
