This is an automated email from the ASF dual-hosted git repository. jerryshao pushed a commit to branch branch-1.3 in repository https://gitbox.apache.org/repos/asf/gravitino.git
commit 07d5936d0a6111f32dbf7dde41c3e032eacbd064 Author: Zhiguo Wu <[email protected]> AuthorDate: Mon Jul 13 15:19:43 2026 +0800 [#11844] feat(tag): Support tags for views and functions (#11897) ### What changes were proposed in this pull request? This PR adds tag support for VIEW and FUNCTION metadata objects. The changes include: - Allowing VIEW and FUNCTION in `TagManager` supported metadata object types. - Updating OpenAPI definitions for metadata object tag operations. - Cleaning up function tag relations when functions are dropped. - Cleaning up view/function tag relations during catalog and schema cascade deletion, including PostgreSQL-specific SQL providers. - Adding unit tests for tag association and cascade cleanup behavior. ### Why are the changes needed? Views and functions are first-class metadata objects in Gravitino, but they were excluded from tag association support. Users could not organize, classify, or discover views and functions through tags like other metadata objects. Fix: #11844 ### Does this PR introduce _any_ user-facing change? Yes. Users can now associate, list, get, and remove tags for VIEW and FUNCTION metadata objects through the existing metadata object tag REST APIs. ### How was this patch tested? Added unit tests covering: - TagManager support for VIEW and FUNCTION. - REST metadata object tag operations for VIEW and FUNCTION. - Tag relation cleanup when views/functions are dropped. - Catalog/schema cascade cleanup for view/function tag relations. - PostgreSQL tag relation cascade SQL provider coverage. Ran: ```bash ./gradlew :core:test --tests org.apache.gravitino.storage.relational.mapper.provider.postgresql.TestTagMetadataObjectRelPostgreSQLProvider --tests org.apache.gravitino.storage.relational.service.TestSchemaMetaService --tests org.apache.gravitino.storage.relational.service.TestCatalogMetaService ``` (cherry picked from commit b464820eb0ca1d51ec954a368c7e6e256a8fceb5) --- .../base/TagMetadataObjectRelBaseSQLProvider.java | 28 +++ .../TagMetadataObjectRelPostgreSQLProvider.java | 28 +++ .../relational/service/FunctionMetaService.java | 6 + .../java/org/apache/gravitino/tag/TagManager.java | 4 +- .../relational/service/TestCatalogMetaService.java | 157 ++++++++++++ .../service/TestFunctionMetaService.java | 42 ++++ .../relational/service/TestSchemaMetaService.java | 146 ++++++++++++ .../relational/service/TestViewMetaService.java | 41 ++++ .../org/apache/gravitino/tag/TestTagManager.java | 167 ++++++++++++- .../gravitino/utils/TestMetadataObjectUtil.java | 11 + docs/open-api/openapi.yaml | 3 +- docs/open-api/tags.yaml | 4 +- .../web/rest/TestMetadataObjectTagOperations.java | 262 ++++++++++++++++++++- 13 files changed, 891 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/TagMetadataObjectRelBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/TagMetadataObjectRelBaseSQLProvider.java index ed6aca6f11..1f3a727066 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/TagMetadataObjectRelBaseSQLProvider.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/TagMetadataObjectRelBaseSQLProvider.java @@ -21,6 +21,7 @@ package org.apache.gravitino.storage.relational.mapper.provider.base; import java.util.List; import org.apache.gravitino.storage.relational.mapper.CatalogMetaMapper; import org.apache.gravitino.storage.relational.mapper.FilesetMetaMapper; +import org.apache.gravitino.storage.relational.mapper.FunctionMetaMapper; import org.apache.gravitino.storage.relational.mapper.MetalakeMetaMapper; import org.apache.gravitino.storage.relational.mapper.ModelMetaMapper; import org.apache.gravitino.storage.relational.mapper.SchemaMetaMapper; @@ -29,6 +30,7 @@ import org.apache.gravitino.storage.relational.mapper.TableMetaMapper; import org.apache.gravitino.storage.relational.mapper.TagMetaMapper; import org.apache.gravitino.storage.relational.mapper.TagMetadataObjectRelMapper; import org.apache.gravitino.storage.relational.mapper.TopicMetaMapper; +import org.apache.gravitino.storage.relational.mapper.ViewMetaMapper; import org.apache.gravitino.storage.relational.po.TagMetadataObjectRelPO; import org.apache.ibatis.annotations.Param; @@ -204,6 +206,16 @@ public class TagMetadataObjectRelBaseSQLProvider { + ModelMetaMapper.TABLE_NAME + " mt WHERE mt.catalog_id = #{catalogId} AND" + " mt.model_id = tmt.metadata_object_id AND tmt.metadata_object_type = 'MODEL'" + + " UNION" + + " SELECT vt.catalog_id FROM " + + ViewMetaMapper.TABLE_NAME + + " vt WHERE vt.catalog_id = #{catalogId} AND" + + " vt.view_id = tmt.metadata_object_id AND tmt.metadata_object_type = 'VIEW'" + + " UNION" + + " SELECT fnt.catalog_id FROM " + + FunctionMetaMapper.TABLE_NAME + + " fnt WHERE fnt.catalog_id = #{catalogId} AND" + + " fnt.function_id = tmt.metadata_object_id AND tmt.metadata_object_type = 'FUNCTION'" + ")"; } @@ -262,6 +274,22 @@ public class TagMetadataObjectRelBaseSQLProvider { + "#{schemaId}" + "</foreach>" + " AND mt.model_id = tmt.metadata_object_id AND tmt.metadata_object_type = 'MODEL'" + + " UNION" + + " SELECT vt.schema_id FROM " + + ViewMetaMapper.TABLE_NAME + + " vt WHERE vt.schema_id IN " + + "<foreach collection='schemaIds' item='schemaId' open='(' close=')' separator=','>" + + "#{schemaId}" + + "</foreach>" + + " AND vt.view_id = tmt.metadata_object_id AND tmt.metadata_object_type = 'VIEW'" + + " UNION" + + " SELECT fnt.schema_id FROM " + + FunctionMetaMapper.TABLE_NAME + + " fnt WHERE fnt.schema_id IN " + + "<foreach collection='schemaIds' item='schemaId' open='(' close=')' separator=','>" + + "#{schemaId}" + + "</foreach>" + + " AND fnt.function_id = tmt.metadata_object_id AND tmt.metadata_object_type = 'FUNCTION'" + ")" + "</script>"; } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/TagMetadataObjectRelPostgreSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/TagMetadataObjectRelPostgreSQLProvider.java index 1bb6a7559c..992e105ee6 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/TagMetadataObjectRelPostgreSQLProvider.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/TagMetadataObjectRelPostgreSQLProvider.java @@ -23,6 +23,7 @@ import static org.apache.gravitino.storage.relational.mapper.TagMetadataObjectRe import java.util.List; import org.apache.gravitino.storage.relational.mapper.CatalogMetaMapper; import org.apache.gravitino.storage.relational.mapper.FilesetMetaMapper; +import org.apache.gravitino.storage.relational.mapper.FunctionMetaMapper; import org.apache.gravitino.storage.relational.mapper.MetalakeMetaMapper; import org.apache.gravitino.storage.relational.mapper.ModelMetaMapper; import org.apache.gravitino.storage.relational.mapper.SchemaMetaMapper; @@ -31,6 +32,7 @@ import org.apache.gravitino.storage.relational.mapper.TableMetaMapper; import org.apache.gravitino.storage.relational.mapper.TagMetaMapper; import org.apache.gravitino.storage.relational.mapper.TagMetadataObjectRelMapper; import org.apache.gravitino.storage.relational.mapper.TopicMetaMapper; +import org.apache.gravitino.storage.relational.mapper.ViewMetaMapper; import org.apache.gravitino.storage.relational.mapper.provider.base.TagMetadataObjectRelBaseSQLProvider; import org.apache.ibatis.annotations.Param; @@ -111,6 +113,16 @@ public class TagMetadataObjectRelPostgreSQLProvider extends TagMetadataObjectRel + ModelMetaMapper.TABLE_NAME + " mt WHERE mt.catalog_id = #{catalogId} AND" + " mt.model_id = tmt.metadata_object_id AND tmt.metadata_object_type = 'MODEL'" + + " UNION" + + " SELECT vt.catalog_id FROM " + + ViewMetaMapper.TABLE_NAME + + " vt WHERE vt.catalog_id = #{catalogId} AND" + + " vt.view_id = tmt.metadata_object_id AND tmt.metadata_object_type = 'VIEW'" + + " UNION" + + " SELECT fnt.catalog_id FROM " + + FunctionMetaMapper.TABLE_NAME + + " fnt WHERE fnt.catalog_id = #{catalogId} AND" + + " fnt.function_id = tmt.metadata_object_id AND tmt.metadata_object_type = 'FUNCTION'" + ")"; } @@ -169,6 +181,22 @@ public class TagMetadataObjectRelPostgreSQLProvider extends TagMetadataObjectRel + "#{schemaId}" + "</foreach>" + " AND mt.model_id = tmt.metadata_object_id AND tmt.metadata_object_type = 'MODEL'" + + " UNION" + + " SELECT vt.schema_id FROM " + + ViewMetaMapper.TABLE_NAME + + " vt WHERE vt.schema_id IN " + + "<foreach collection='schemaIds' item='schemaId' open='(' close=')' separator=','>" + + "#{schemaId}" + + "</foreach>" + + " AND vt.view_id = tmt.metadata_object_id AND tmt.metadata_object_type = 'VIEW'" + + " UNION" + + " SELECT fnt.schema_id FROM " + + FunctionMetaMapper.TABLE_NAME + + " fnt WHERE fnt.schema_id IN " + + "<foreach collection='schemaIds' item='schemaId' open='(' close=')' separator=','>" + + "#{schemaId}" + + "</foreach>" + + " AND fnt.function_id = tmt.metadata_object_id AND tmt.metadata_object_type = 'FUNCTION'" + ")" + "</script>"; } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java index 33532edad2..2c582dc8c0 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java @@ -45,6 +45,7 @@ import org.apache.gravitino.storage.relational.mapper.FunctionMetaMapper; import org.apache.gravitino.storage.relational.mapper.FunctionVersionMetaMapper; import org.apache.gravitino.storage.relational.mapper.OwnerMetaMapper; import org.apache.gravitino.storage.relational.mapper.SecurableObjectMapper; +import org.apache.gravitino.storage.relational.mapper.TagMetadataObjectRelMapper; import org.apache.gravitino.storage.relational.po.FunctionMaxVersionPO; import org.apache.gravitino.storage.relational.po.FunctionPO; import org.apache.gravitino.storage.relational.utils.ExceptionUtils; @@ -166,6 +167,11 @@ public class FunctionMetaService { mapper -> mapper.softDeleteObjectRelsByMetadataObject( functionId, MetadataObject.Type.FUNCTION.name())); + SessionUtils.doWithoutCommit( + TagMetadataObjectRelMapper.class, + mapper -> + mapper.softDeleteTagMetadataObjectRelsByMetadataObject( + functionId, MetadataObject.Type.FUNCTION.name())); } }); diff --git a/core/src/main/java/org/apache/gravitino/tag/TagManager.java b/core/src/main/java/org/apache/gravitino/tag/TagManager.java index eb26cf37e9..49b8e112b4 100644 --- a/core/src/main/java/org/apache/gravitino/tag/TagManager.java +++ b/core/src/main/java/org/apache/gravitino/tag/TagManager.java @@ -69,10 +69,12 @@ public class TagManager implements TagDispatcher { MetadataObject.Type.CATALOG, MetadataObject.Type.SCHEMA, MetadataObject.Type.TABLE, + MetadataObject.Type.VIEW, MetadataObject.Type.FILESET, MetadataObject.Type.TOPIC, MetadataObject.Type.COLUMN, - MetadataObject.Type.MODEL); + MetadataObject.Type.MODEL, + MetadataObject.Type.FUNCTION); public TagManager(IdGenerator idGenerator, EntityStore entityStore) { this.idGenerator = idGenerator; diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestCatalogMetaService.java b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestCatalogMetaService.java index 589dbec14e..de5520908a 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestCatalogMetaService.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestCatalogMetaService.java @@ -24,19 +24,37 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; import java.time.Instant; import java.util.List; import org.apache.gravitino.Catalog; import org.apache.gravitino.Entity; import org.apache.gravitino.EntityAlreadyExistsException; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; import org.apache.gravitino.meta.AuditInfo; import org.apache.gravitino.meta.CatalogEntity; +import org.apache.gravitino.meta.ColumnEntity; +import org.apache.gravitino.meta.FilesetEntity; +import org.apache.gravitino.meta.FunctionEntity; +import org.apache.gravitino.meta.ModelEntity; +import org.apache.gravitino.meta.SchemaEntity; +import org.apache.gravitino.meta.TableEntity; +import org.apache.gravitino.meta.TagEntity; +import org.apache.gravitino.meta.TopicEntity; +import org.apache.gravitino.meta.ViewEntity; +import org.apache.gravitino.rel.types.Types; import org.apache.gravitino.storage.RandomIdGenerator; import org.apache.gravitino.storage.relational.TestJDBCBackend; import org.apache.gravitino.storage.relational.mapper.CatalogMetaMapper; +import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; import org.apache.gravitino.storage.relational.utils.SessionUtils; import org.apache.gravitino.utils.NameIdentifierUtil; import org.apache.gravitino.utils.NamespaceUtil; +import org.apache.ibatis.session.SqlSession; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.TestTemplate; @@ -176,4 +194,143 @@ public class TestCatalogMetaService extends TestJDBCBackend { backend.hardDeleteLegacyData(Entity.EntityType.CATALOG, Instant.now().toEpochMilli() + 3000); assertFalse(legacyRecordExistsInDB(catalog.id(), Entity.EntityType.CATALOG)); } + + @TestTemplate + public void testDeleteCatalogCascadeRemovesTagRelations() throws IOException { + CatalogEntity catalog = + createCatalog( + RandomIdGenerator.INSTANCE.nextId(), + NamespaceUtil.ofCatalog(metalakeName), + "catalog_with_tags", + auditInfo); + backend.insert(catalog, false); + + SchemaEntity schema = + createSchemaEntity( + RandomIdGenerator.INSTANCE.nextId(), + NamespaceUtil.ofSchema(metalakeName, catalog.name()), + "schema_with_tags", + AUDIT_INFO); + backend.insert(schema, false); + + Namespace objectNamespace = Namespace.of(metalakeName, catalog.name(), schema.name()); + ColumnEntity column = + ColumnEntity.builder() + .withId(RandomIdGenerator.INSTANCE.nextId()) + .withName("column_with_tag") + .withPosition(0) + .withAutoIncrement(false) + .withNullable(false) + .withDataType(Types.IntegerType.get()) + .withAuditInfo(AUDIT_INFO) + .build(); + TableEntity table = + TableEntity.builder() + .withId(RandomIdGenerator.INSTANCE.nextId()) + .withName("table_with_tag") + .withNamespace(objectNamespace) + .withColumns(List.of(column)) + .withAuditInfo(AUDIT_INFO) + .build(); + TableMetaService.getInstance().insertTable(table, false); + TopicEntity topic = + createTopicEntity( + RandomIdGenerator.INSTANCE.nextId(), objectNamespace, "topic_with_tag", AUDIT_INFO); + TopicMetaService.getInstance().insertTopic(topic, false); + FilesetEntity fileset = + createFilesetEntity( + RandomIdGenerator.INSTANCE.nextId(), objectNamespace, "fileset_with_tag", AUDIT_INFO); + FilesetMetaService.getInstance().insertFileset(fileset, false); + ModelEntity model = + createModelEntity( + RandomIdGenerator.INSTANCE.nextId(), + objectNamespace, + "model_with_tag", + "comment", + 1, + null, + AUDIT_INFO); + ModelMetaService.getInstance().insertModel(model, false); + ViewEntity view = + createViewEntity(RandomIdGenerator.INSTANCE.nextId(), objectNamespace, "view_with_tag"); + FunctionEntity function = + createFunctionEntity( + RandomIdGenerator.INSTANCE.nextId(), objectNamespace, "function_with_tag", AUDIT_INFO); + ViewMetaService.getInstance().insertView(view, false); + FunctionMetaService.getInstance().insertFunction(function, false); + + TagEntity tag = + TagEntity.builder() + .withId(RandomIdGenerator.INSTANCE.nextId()) + .withName("tag1") + .withNamespace(NamespaceUtil.ofTag(metalakeName)) + .withAuditInfo(AUDIT_INFO) + .build(); + TagMetaService.getInstance().insertTag(tag, false); + associateTag(tag, catalog.nameIdentifier(), catalog.type()); + associateTag(tag, schema.nameIdentifier(), schema.type()); + associateTag(tag, table.nameIdentifier(), table.type()); + associateTag( + tag, + NameIdentifier.of(Namespace.fromString(table.nameIdentifier().toString()), column.name()), + column.type()); + associateTag(tag, topic.nameIdentifier(), topic.type()); + associateTag(tag, fileset.nameIdentifier(), fileset.type()); + associateTag(tag, model.nameIdentifier(), model.type()); + associateTag(tag, view.nameIdentifier(), view.type()); + associateTag(tag, function.nameIdentifier(), function.type()); + + assertEquals(1, countActiveTagRelForMetadataObject(catalog.id(), "CATALOG")); + assertEquals(1, countActiveTagRelForMetadataObject(schema.id(), "SCHEMA")); + assertEquals(1, countActiveTagRelForMetadataObject(table.id(), "TABLE")); + assertEquals(1, countActiveTagRelForMetadataObject(column.id(), "COLUMN")); + assertEquals(1, countActiveTagRelForMetadataObject(topic.id(), "TOPIC")); + assertEquals(1, countActiveTagRelForMetadataObject(fileset.id(), "FILESET")); + assertEquals(1, countActiveTagRelForMetadataObject(model.id(), "MODEL")); + assertEquals(1, countActiveTagRelForMetadataObject(view.id(), "VIEW")); + assertEquals(1, countActiveTagRelForMetadataObject(function.id(), "FUNCTION")); + + assertTrue(CatalogMetaService.getInstance().deleteCatalog(catalog.nameIdentifier(), true)); + + assertEquals(0, countActiveTagRelForMetadataObject(catalog.id(), "CATALOG")); + assertEquals(0, countActiveTagRelForMetadataObject(schema.id(), "SCHEMA")); + assertEquals(0, countActiveTagRelForMetadataObject(table.id(), "TABLE")); + assertEquals(0, countActiveTagRelForMetadataObject(column.id(), "COLUMN")); + assertEquals(0, countActiveTagRelForMetadataObject(topic.id(), "TOPIC")); + assertEquals(0, countActiveTagRelForMetadataObject(fileset.id(), "FILESET")); + assertEquals(0, countActiveTagRelForMetadataObject(model.id(), "MODEL")); + assertEquals(0, countActiveTagRelForMetadataObject(view.id(), "VIEW")); + assertEquals(0, countActiveTagRelForMetadataObject(function.id(), "FUNCTION")); + } + + private void associateTag(TagEntity tag, NameIdentifier ident, Entity.EntityType type) + throws IOException { + TagMetaService.getInstance() + .associateTagsWithMetadataObject( + ident, + type, + new NameIdentifier[] {NameIdentifierUtil.ofTag(metalakeName, tag.name())}, + new NameIdentifier[0]); + } + + private int countActiveTagRelForMetadataObject(Long metadataObjectId, String metadataObjectType) { + try (SqlSession sqlSession = + SqlSessionFactoryHelper.getInstance().getSqlSessionFactory().openSession(true); + Connection connection = sqlSession.getConnection(); + Statement statement = connection.createStatement(); + ResultSet rs = + statement.executeQuery( + String.format( + "SELECT count(*) FROM tag_relation_meta" + + " WHERE metadata_object_id = %d AND metadata_object_type = '%s'" + + " AND deleted_at = 0", + metadataObjectId, metadataObjectType))) { + if (rs.next()) { + return rs.getInt(1); + } + throw new RuntimeException("No result for countActiveTagRelForMetadataObject"); + } catch (SQLException e) { + throw new RuntimeException("SQL execution failed", e); + } + } } diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestFunctionMetaService.java b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestFunctionMetaService.java index 5d2471c9f3..fe5d46db8b 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestFunctionMetaService.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestFunctionMetaService.java @@ -47,6 +47,7 @@ import org.apache.gravitino.exceptions.NoSuchEntityException; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.gravitino.meta.FunctionEntity; import org.apache.gravitino.meta.RoleEntity; +import org.apache.gravitino.meta.TagEntity; import org.apache.gravitino.meta.UserEntity; import org.apache.gravitino.storage.RandomIdGenerator; import org.apache.gravitino.storage.relational.TestJDBCBackend; @@ -280,6 +281,23 @@ public class TestFunctionMetaService extends TestJDBCBackend { ImmutableMap.of()); RoleMetaService.getInstance().insertRole(role, false); + // Set up tag relation + TagEntity tag = + TagEntity.builder() + .withId(RandomIdGenerator.INSTANCE.nextId()) + .withName("tag1") + .withNamespace(NamespaceUtil.ofTag(metalakeName)) + .withAuditInfo(AUDIT_INFO) + .build(); + TagMetaService.getInstance().insertTag(tag, false); + TagMetaService.getInstance() + .associateTagsWithMetadataObject( + function.nameIdentifier(), + function.type(), + new NameIdentifier[] {NameIdentifierUtil.ofTag(metalakeName, tag.name())}, + new NameIdentifier[0]); + assertEquals(1, countActiveTagRelForMetadataObject(function.id(), "FUNCTION")); + NameIdentifier functionIdent = NameIdentifier.of(metalakeName, catalogName, schemaName, functionName); assertTrue(FunctionMetaService.getInstance().deleteFunction(functionIdent)); @@ -294,6 +312,9 @@ public class TestFunctionMetaService extends TestJDBCBackend { // Verify securable object (role) relation is cleaned up assertEquals(0, countActiveObjectRelForRole(role.id())); + + // Verify tag relation is cleaned up + assertEquals(0, countActiveTagRelForMetadataObject(function.id(), "FUNCTION")); } @TestTemplate @@ -568,6 +589,27 @@ public class TestFunctionMetaService extends TestJDBCBackend { } } + private int countActiveTagRelForMetadataObject(Long metadataObjectId, String metadataObjectType) { + try (SqlSession sqlSession = + SqlSessionFactoryHelper.getInstance().getSqlSessionFactory().openSession(true); + Connection connection = sqlSession.getConnection(); + Statement statement = connection.createStatement(); + ResultSet rs = + statement.executeQuery( + String.format( + "SELECT count(*) FROM tag_relation_meta" + + " WHERE metadata_object_id = %d AND metadata_object_type = '%s'" + + " AND deleted_at = 0", + metadataObjectId, metadataObjectType))) { + if (rs.next()) { + return rs.getInt(1); + } + throw new RuntimeException("No result for countActiveTagRelForMetadataObject"); + } catch (SQLException e) { + throw new RuntimeException("SQL execution failed", e); + } + } + private Map<Integer, Long> listFunctionVersions(Long functionId) { Map<Integer, Long> versionDeletedTime = new HashMap<>(); try (SqlSession sqlSession = diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestSchemaMetaService.java b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestSchemaMetaService.java index d11efa6950..a1e43144ec 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestSchemaMetaService.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestSchemaMetaService.java @@ -23,6 +23,10 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; import java.time.Instant; import java.util.Collections; import java.util.List; @@ -31,13 +35,24 @@ import java.util.stream.Collectors; import org.apache.gravitino.Entity; import org.apache.gravitino.EntityAlreadyExistsException; import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; import org.apache.gravitino.exceptions.NonEmptyEntityException; +import org.apache.gravitino.meta.ColumnEntity; +import org.apache.gravitino.meta.FilesetEntity; +import org.apache.gravitino.meta.FunctionEntity; +import org.apache.gravitino.meta.ModelEntity; import org.apache.gravitino.meta.SchemaEntity; +import org.apache.gravitino.meta.TableEntity; +import org.apache.gravitino.meta.TagEntity; import org.apache.gravitino.meta.TopicEntity; +import org.apache.gravitino.meta.ViewEntity; +import org.apache.gravitino.rel.types.Types; import org.apache.gravitino.storage.RandomIdGenerator; import org.apache.gravitino.storage.relational.TestJDBCBackend; +import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; import org.apache.gravitino.utils.NameIdentifierUtil; import org.apache.gravitino.utils.NamespaceUtil; +import org.apache.ibatis.session.SqlSession; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.TestTemplate; @@ -323,6 +338,106 @@ public class TestSchemaMetaService extends TestJDBCBackend { NameIdentifier.of(metalakeName, catalogName, "anc_a"), Entity.EntityType.SCHEMA)); } + @TestTemplate + public void testDeleteSchemaCascadeRemovesTagRelations() throws IOException { + createAndInsertMakeLake(metalakeName); + createAndInsertCatalog(metalakeName, catalogName); + + SchemaEntity schema = + createSchemaEntity( + RandomIdGenerator.INSTANCE.nextId(), + NamespaceUtil.ofSchema(metalakeName, catalogName), + "schema_with_tags", + AUDIT_INFO); + SchemaMetaService.getInstance().insertSchema(schema, false); + + Namespace objectNamespace = Namespace.of(metalakeName, catalogName, schema.name()); + ColumnEntity column = + ColumnEntity.builder() + .withId(RandomIdGenerator.INSTANCE.nextId()) + .withName("column_with_tag") + .withPosition(0) + .withAutoIncrement(false) + .withNullable(false) + .withDataType(Types.IntegerType.get()) + .withAuditInfo(AUDIT_INFO) + .build(); + TableEntity table = + TableEntity.builder() + .withId(RandomIdGenerator.INSTANCE.nextId()) + .withName("table_with_tag") + .withNamespace(objectNamespace) + .withColumns(List.of(column)) + .withAuditInfo(AUDIT_INFO) + .build(); + TableMetaService.getInstance().insertTable(table, false); + TopicEntity topic = + createTopicEntity( + RandomIdGenerator.INSTANCE.nextId(), objectNamespace, "topic_with_tag", AUDIT_INFO); + TopicMetaService.getInstance().insertTopic(topic, false); + FilesetEntity fileset = + createFilesetEntity( + RandomIdGenerator.INSTANCE.nextId(), objectNamespace, "fileset_with_tag", AUDIT_INFO); + FilesetMetaService.getInstance().insertFileset(fileset, false); + ModelEntity model = + createModelEntity( + RandomIdGenerator.INSTANCE.nextId(), + objectNamespace, + "model_with_tag", + "comment", + 1, + null, + AUDIT_INFO); + ModelMetaService.getInstance().insertModel(model, false); + ViewEntity view = + createViewEntity(RandomIdGenerator.INSTANCE.nextId(), objectNamespace, "view_with_tag"); + FunctionEntity function = + createFunctionEntity( + RandomIdGenerator.INSTANCE.nextId(), objectNamespace, "function_with_tag", AUDIT_INFO); + ViewMetaService.getInstance().insertView(view, false); + FunctionMetaService.getInstance().insertFunction(function, false); + + TagEntity tag = + TagEntity.builder() + .withId(RandomIdGenerator.INSTANCE.nextId()) + .withName("tag1") + .withNamespace(NamespaceUtil.ofTag(metalakeName)) + .withAuditInfo(AUDIT_INFO) + .build(); + TagMetaService.getInstance().insertTag(tag, false); + associateTag(tag, schema.nameIdentifier(), schema.type()); + associateTag(tag, table.nameIdentifier(), table.type()); + associateTag( + tag, + NameIdentifier.of(Namespace.fromString(table.nameIdentifier().toString()), column.name()), + column.type()); + associateTag(tag, topic.nameIdentifier(), topic.type()); + associateTag(tag, fileset.nameIdentifier(), fileset.type()); + associateTag(tag, model.nameIdentifier(), model.type()); + associateTag(tag, view.nameIdentifier(), view.type()); + associateTag(tag, function.nameIdentifier(), function.type()); + + Assertions.assertEquals(1, countActiveTagRelForMetadataObject(schema.id(), "SCHEMA")); + Assertions.assertEquals(1, countActiveTagRelForMetadataObject(table.id(), "TABLE")); + Assertions.assertEquals(1, countActiveTagRelForMetadataObject(column.id(), "COLUMN")); + Assertions.assertEquals(1, countActiveTagRelForMetadataObject(topic.id(), "TOPIC")); + Assertions.assertEquals(1, countActiveTagRelForMetadataObject(fileset.id(), "FILESET")); + Assertions.assertEquals(1, countActiveTagRelForMetadataObject(model.id(), "MODEL")); + Assertions.assertEquals(1, countActiveTagRelForMetadataObject(view.id(), "VIEW")); + Assertions.assertEquals(1, countActiveTagRelForMetadataObject(function.id(), "FUNCTION")); + + assertTrue(SchemaMetaService.getInstance().deleteSchema(schema.nameIdentifier(), true)); + + Assertions.assertEquals(0, countActiveTagRelForMetadataObject(schema.id(), "SCHEMA")); + Assertions.assertEquals(0, countActiveTagRelForMetadataObject(table.id(), "TABLE")); + Assertions.assertEquals(0, countActiveTagRelForMetadataObject(column.id(), "COLUMN")); + Assertions.assertEquals(0, countActiveTagRelForMetadataObject(topic.id(), "TOPIC")); + Assertions.assertEquals(0, countActiveTagRelForMetadataObject(fileset.id(), "FILESET")); + Assertions.assertEquals(0, countActiveTagRelForMetadataObject(model.id(), "MODEL")); + Assertions.assertEquals(0, countActiveTagRelForMetadataObject(view.id(), "VIEW")); + Assertions.assertEquals(0, countActiveTagRelForMetadataObject(function.id(), "FUNCTION")); + } + @TestTemplate public void testDeleteHierarchicalSchemaCascadeEscapesLikeMetacharacters() throws IOException { createAndInsertMakeLake(metalakeName); @@ -423,4 +538,35 @@ public class TestSchemaMetaService extends TestJDBCBackend { .getSchemaByIdentifier(NameIdentifier.of(metalakeName, catalogName, ancestorAB)) .id()); } + + private void associateTag(TagEntity tag, NameIdentifier ident, Entity.EntityType type) + throws IOException { + TagMetaService.getInstance() + .associateTagsWithMetadataObject( + ident, + type, + new NameIdentifier[] {NameIdentifierUtil.ofTag(metalakeName, tag.name())}, + new NameIdentifier[0]); + } + + private int countActiveTagRelForMetadataObject(Long metadataObjectId, String metadataObjectType) { + try (SqlSession sqlSession = + SqlSessionFactoryHelper.getInstance().getSqlSessionFactory().openSession(true); + Connection connection = sqlSession.getConnection(); + Statement statement = connection.createStatement(); + ResultSet rs = + statement.executeQuery( + String.format( + "SELECT count(*) FROM tag_relation_meta" + + " WHERE metadata_object_id = %d AND metadata_object_type = '%s'" + + " AND deleted_at = 0", + metadataObjectId, metadataObjectType))) { + if (rs.next()) { + return rs.getInt(1); + } + throw new RuntimeException("No result for countActiveTagRelForMetadataObject"); + } catch (SQLException e) { + throw new RuntimeException("SQL execution failed", e); + } + } } diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestViewMetaService.java b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestViewMetaService.java index 2c6488e329..edeaff2565 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestViewMetaService.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestViewMetaService.java @@ -39,6 +39,7 @@ import org.apache.gravitino.Namespace; import org.apache.gravitino.exceptions.NoSuchEntityException; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.gravitino.meta.AuditInfo; +import org.apache.gravitino.meta.TagEntity; import org.apache.gravitino.meta.ViewEntity; import org.apache.gravitino.rel.Column; import org.apache.gravitino.rel.Representation; @@ -47,6 +48,7 @@ import org.apache.gravitino.rel.types.Types; import org.apache.gravitino.storage.RandomIdGenerator; import org.apache.gravitino.storage.relational.TestJDBCBackend; import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import org.apache.gravitino.utils.NameIdentifierUtil; import org.apache.gravitino.utils.NamespaceUtil; import org.apache.ibatis.session.SqlSession; import org.junit.jupiter.api.BeforeEach; @@ -208,12 +210,30 @@ public class TestViewMetaService extends TestJDBCBackend { ViewMetaService.getInstance().insertView(view, false); + // Set up tag relation + TagEntity tag = + TagEntity.builder() + .withId(RandomIdGenerator.INSTANCE.nextId()) + .withName("tag1") + .withNamespace(NamespaceUtil.ofTag(metalakeName)) + .withAuditInfo(AUDIT_INFO) + .build(); + TagMetaService.getInstance().insertTag(tag, false); + TagMetaService.getInstance() + .associateTagsWithMetadataObject( + view.nameIdentifier(), + view.type(), + new NameIdentifier[] {NameIdentifierUtil.ofTag(metalakeName, tag.name())}, + new NameIdentifier[0]); + assertEquals(1, countActiveTagRelForMetadataObject(view.id(), "VIEW")); + NameIdentifier viewIdent = NameIdentifier.of(metalakeName, catalogName, schemaName, viewName); assertTrue(ViewMetaService.getInstance().deleteView(viewIdent)); assertThrows( NoSuchEntityException.class, () -> ViewMetaService.getInstance().getViewByIdentifier(viewIdent)); + assertEquals(0, countActiveTagRelForMetadataObject(view.id(), "VIEW")); } @TestTemplate @@ -333,4 +353,25 @@ public class TestViewMetaService extends TestJDBCBackend { } return versionDeletedTime; } + + private int countActiveTagRelForMetadataObject(Long metadataObjectId, String metadataObjectType) { + try (SqlSession sqlSession = + SqlSessionFactoryHelper.getInstance().getSqlSessionFactory().openSession(true); + Connection connection = sqlSession.getConnection(); + Statement statement = connection.createStatement(); + ResultSet rs = + statement.executeQuery( + String.format( + "SELECT count(*) FROM tag_relation_meta" + + " WHERE metadata_object_id = %d AND metadata_object_type = '%s'" + + " AND deleted_at = 0", + metadataObjectId, metadataObjectType))) { + if (rs.next()) { + return rs.getInt(1); + } + throw new RuntimeException("No result for countActiveTagRelForMetadataObject"); + } catch (SQLException e) { + throw new RuntimeException("SQL execution failed", e); + } + } } diff --git a/core/src/test/java/org/apache/gravitino/tag/TestTagManager.java b/core/src/test/java/org/apache/gravitino/tag/TestTagManager.java index ae377b8ad0..593e42f5ad 100644 --- a/core/src/test/java/org/apache/gravitino/tag/TestTagManager.java +++ b/core/src/test/java/org/apache/gravitino/tag/TestTagManager.java @@ -62,22 +62,36 @@ import org.apache.gravitino.GravitinoEnv; import org.apache.gravitino.MetadataObject; import org.apache.gravitino.Namespace; import org.apache.gravitino.catalog.CatalogDispatcher; +import org.apache.gravitino.catalog.FunctionDispatcher; import org.apache.gravitino.catalog.SchemaDispatcher; import org.apache.gravitino.catalog.TableDispatcher; +import org.apache.gravitino.catalog.ViewDispatcher; import org.apache.gravitino.exceptions.NoSuchMetalakeException; import org.apache.gravitino.exceptions.NoSuchTagException; import org.apache.gravitino.exceptions.NotFoundException; import org.apache.gravitino.exceptions.TagAlreadyAssociatedException; import org.apache.gravitino.exceptions.TagAlreadyExistsException; +import org.apache.gravitino.function.FunctionDefinition; +import org.apache.gravitino.function.FunctionDefinitions; +import org.apache.gravitino.function.FunctionImpl; +import org.apache.gravitino.function.FunctionImpls; +import org.apache.gravitino.function.FunctionParam; +import org.apache.gravitino.function.FunctionParams; +import org.apache.gravitino.function.FunctionType; import org.apache.gravitino.lock.LockManager; import org.apache.gravitino.meta.AuditInfo; import org.apache.gravitino.meta.BaseMetalake; import org.apache.gravitino.meta.CatalogEntity; import org.apache.gravitino.meta.ColumnEntity; +import org.apache.gravitino.meta.FunctionEntity; import org.apache.gravitino.meta.SchemaEntity; import org.apache.gravitino.meta.SchemaVersion; import org.apache.gravitino.meta.TableEntity; +import org.apache.gravitino.meta.ViewEntity; import org.apache.gravitino.metalake.MetalakeDispatcher; +import org.apache.gravitino.rel.Column; +import org.apache.gravitino.rel.Representation; +import org.apache.gravitino.rel.SQLRepresentation; import org.apache.gravitino.rel.types.Types; import org.apache.gravitino.storage.IdGenerator; import org.apache.gravitino.storage.RandomIdGenerator; @@ -107,10 +121,16 @@ public class TestTagManager { private static final String COLUMN = "column_for_tag_test"; + private static final String VIEW = "view_for_tag_test"; + + private static final String FUNCTION = "function_for_tag_test"; + private static final MetalakeDispatcher metalakeDispatcher = mock(MetalakeDispatcher.class); private static final CatalogDispatcher catalogDispatcher = mock(CatalogDispatcher.class); private static final SchemaDispatcher schemaDispatcher = mock(SchemaDispatcher.class); private static final TableDispatcher tableDispatcher = mock(TableDispatcher.class); + private static final ViewDispatcher viewDispatcher = mock(ViewDispatcher.class); + private static final FunctionDispatcher functionDispatcher = mock(FunctionDispatcher.class); private static EntityStore entityStore; @@ -211,6 +231,37 @@ public class TestTagManager { .build(); entityStore.put(table, false /* overwritten */); + ViewEntity view = + ViewEntity.builder() + .withId(idGenerator.nextId()) + .withName(VIEW) + .withNamespace(Namespace.of(METALAKE, CATALOG, SCHEMA)) + .withColumns(new Column[0]) + .withRepresentations( + new Representation[] { + SQLRepresentation.builder().withDialect("unknown").withSql("SELECT 1").build() + }) + .withAuditInfo(audit) + .build(); + entityStore.put(view, false /* overwritten */); + + FunctionParam param = FunctionParams.of("param", Types.IntegerType.get()); + FunctionImpl impl = FunctionImpls.ofJava(FunctionImpl.RuntimeType.SPARK, "mock.udf.class.name"); + FunctionDefinition definition = + FunctionDefinitions.of( + new FunctionParam[] {param}, Types.IntegerType.get(), new FunctionImpl[] {impl}); + FunctionEntity function = + FunctionEntity.builder() + .withId(idGenerator.nextId()) + .withName(FUNCTION) + .withNamespace(Namespace.of(METALAKE, CATALOG, SCHEMA)) + .withFunctionType(FunctionType.SCALAR) + .withDeterministic(false) + .withDefinitions(new FunctionDefinition[] {definition}) + .withAuditInfo(audit) + .build(); + entityStore.put(function, false /* overwritten */); + tagManager = new TagManager(idGenerator, entityStore); FieldUtils.writeField( @@ -218,11 +269,16 @@ public class TestTagManager { FieldUtils.writeField(GravitinoEnv.getInstance(), "catalogDispatcher", catalogDispatcher, true); FieldUtils.writeField(GravitinoEnv.getInstance(), "schemaDispatcher", schemaDispatcher, true); FieldUtils.writeField(GravitinoEnv.getInstance(), "tableDispatcher", tableDispatcher, true); + FieldUtils.writeField(GravitinoEnv.getInstance(), "viewDispatcher", viewDispatcher, true); + FieldUtils.writeField( + GravitinoEnv.getInstance(), "functionDispatcher", functionDispatcher, true); when(metalakeDispatcher.metalakeExists(any())).thenReturn(true); when(catalogDispatcher.catalogExists(any())).thenReturn(true); when(schemaDispatcher.schemaExists(any())).thenReturn(true); when(tableDispatcher.tableExists(any())).thenReturn(true); + when(viewDispatcher.viewExists(any())).thenReturn(true); + when(functionDispatcher.functionExists(any())).thenReturn(true); } @AfterAll @@ -262,6 +318,19 @@ public class TestTagManager { String[] columnTags = tagManager.listTagsForMetadataObject(METALAKE, columnObject); tagManager.associateTagsForMetadataObject(METALAKE, columnObject, null, columnTags); + MetadataObject viewObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofView(METALAKE, CATALOG, SCHEMA, VIEW), Entity.EntityType.VIEW); + String[] viewTags = tagManager.listTagsForMetadataObject(METALAKE, viewObject); + tagManager.associateTagsForMetadataObject(METALAKE, viewObject, null, viewTags); + + MetadataObject functionObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofFunction(METALAKE, CATALOG, SCHEMA, FUNCTION), + Entity.EntityType.FUNCTION); + String[] functionTags = tagManager.listTagsForMetadataObject(METALAKE, functionObject); + tagManager.associateTagsForMetadataObject(METALAKE, functionObject, null, functionTags); + Arrays.stream(tagManager.listTags(METALAKE)).forEach(n -> tagManager.deleteTag(METALAKE, n)); } @@ -501,6 +570,44 @@ public class TestTagManager { Assertions.assertEquals(2, tags6.length); Assertions.assertEquals(ImmutableSet.of("tag1", "tag3"), ImmutableSet.copyOf(tags6)); + // Test associate tags for view + MetadataObject viewObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofView(METALAKE, CATALOG, SCHEMA, VIEW), Entity.EntityType.VIEW); + String[] tagsForView = + tagManager.associateTagsForMetadataObject(METALAKE, viewObject, tagsToAdd1, null); + + Assertions.assertEquals(1, tagsForView.length); + Assertions.assertEquals(ImmutableSet.of("tag1"), ImmutableSet.copyOf(tagsForView)); + + // Test associate and disassociate same tags for view + String[] tagsForViewAfterUpdate = + tagManager.associateTagsForMetadataObject(METALAKE, viewObject, tagsToAdd2, tagsToRemove1); + + Assertions.assertEquals(2, tagsForViewAfterUpdate.length); + Assertions.assertEquals( + ImmutableSet.of("tag1", "tag3"), ImmutableSet.copyOf(tagsForViewAfterUpdate)); + + // Test associate tags for function + MetadataObject functionObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofFunction(METALAKE, CATALOG, SCHEMA, FUNCTION), + Entity.EntityType.FUNCTION); + String[] tagsForFunction = + tagManager.associateTagsForMetadataObject(METALAKE, functionObject, tagsToAdd1, null); + + Assertions.assertEquals(1, tagsForFunction.length); + Assertions.assertEquals(ImmutableSet.of("tag1"), ImmutableSet.copyOf(tagsForFunction)); + + // Test associate and disassociate same tags for function + String[] tagsForFunctionAfterUpdate = + tagManager.associateTagsForMetadataObject( + METALAKE, functionObject, tagsToAdd2, tagsToRemove1); + + Assertions.assertEquals(2, tagsForFunctionAfterUpdate.length); + Assertions.assertEquals( + ImmutableSet.of("tag1", "tag3"), ImmutableSet.copyOf(tagsForFunctionAfterUpdate)); + // Test associate and disassociate same tags for column MetadataObject columnObject = NameIdentifierUtil.toMetadataObject( @@ -552,6 +659,13 @@ public class TestTagManager { NameIdentifierUtil.toMetadataObject( NameIdentifierUtil.ofColumn(METALAKE, CATALOG, SCHEMA, TABLE, COLUMN), Entity.EntityType.COLUMN); + MetadataObject viewObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofView(METALAKE, CATALOG, SCHEMA, VIEW), Entity.EntityType.VIEW); + MetadataObject functionObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofFunction(METALAKE, CATALOG, SCHEMA, FUNCTION), + Entity.EntityType.FUNCTION); tagManager.associateTagsForMetadataObject( METALAKE, catalogObject, new String[] {tag1.name(), tag2.name(), tag3.name()}, null); @@ -561,11 +675,16 @@ public class TestTagManager { METALAKE, tableObject, new String[] {tag1.name()}, null); tagManager.associateTagsForMetadataObject( METALAKE, columnObject, new String[] {tag1.name()}, null); + tagManager.associateTagsForMetadataObject( + METALAKE, viewObject, new String[] {tag1.name()}, null); + tagManager.associateTagsForMetadataObject( + METALAKE, functionObject, new String[] {tag1.name()}, null); MetadataObject[] objects = tagManager.listMetadataObjectsForTag(METALAKE, tag1.name()); - Assertions.assertEquals(4, objects.length); + Assertions.assertEquals(6, objects.length); Assertions.assertEquals( - ImmutableSet.of(catalogObject, schemaObject, tableObject, columnObject), + ImmutableSet.of( + catalogObject, schemaObject, tableObject, columnObject, viewObject, functionObject), ImmutableSet.copyOf(objects)); MetadataObject[] objects1 = tagManager.listMetadataObjectsForTag(METALAKE, tag2.name()); @@ -607,6 +726,13 @@ public class TestTagManager { NameIdentifierUtil.toMetadataObject( NameIdentifierUtil.ofColumn(METALAKE, CATALOG, SCHEMA, TABLE, COLUMN), Entity.EntityType.COLUMN); + MetadataObject viewObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofView(METALAKE, CATALOG, SCHEMA, VIEW), Entity.EntityType.VIEW); + MetadataObject functionObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofFunction(METALAKE, CATALOG, SCHEMA, FUNCTION), + Entity.EntityType.FUNCTION); tagManager.associateTagsForMetadataObject( METALAKE, catalogObject, new String[] {tag1.name(), tag2.name(), tag3.name()}, null); @@ -616,6 +742,10 @@ public class TestTagManager { METALAKE, tableObject, new String[] {tag1.name()}, null); tagManager.associateTagsForMetadataObject( METALAKE, columnObject, new String[] {tag1.name()}, null); + tagManager.associateTagsForMetadataObject( + METALAKE, viewObject, new String[] {tag1.name()}, null); + tagManager.associateTagsForMetadataObject( + METALAKE, functionObject, new String[] {tag1.name()}, null); String[] tags = tagManager.listTagsForMetadataObject(METALAKE, catalogObject); Assertions.assertEquals(3, tags.length); @@ -649,6 +779,22 @@ public class TestTagManager { Assertions.assertEquals(1, tagsInfo3.length); Assertions.assertEquals(ImmutableSet.of(tag1), ImmutableSet.copyOf(tagsInfo3)); + String[] tags4 = tagManager.listTagsForMetadataObject(METALAKE, viewObject); + Assertions.assertEquals(1, tags4.length); + Assertions.assertEquals(ImmutableSet.of("tag1"), ImmutableSet.copyOf(tags4)); + + Tag[] tagsInfo4 = tagManager.listTagsInfoForMetadataObject(METALAKE, viewObject); + Assertions.assertEquals(1, tagsInfo4.length); + Assertions.assertEquals(ImmutableSet.of(tag1), ImmutableSet.copyOf(tagsInfo4)); + + String[] tags5 = tagManager.listTagsForMetadataObject(METALAKE, functionObject); + Assertions.assertEquals(1, tags5.length); + Assertions.assertEquals(ImmutableSet.of("tag1"), ImmutableSet.copyOf(tags5)); + + Tag[] tagsInfo5 = tagManager.listTagsInfoForMetadataObject(METALAKE, functionObject); + Assertions.assertEquals(1, tagsInfo5.length); + Assertions.assertEquals(ImmutableSet.of(tag1), ImmutableSet.copyOf(tagsInfo5)); + // List tags for non-existent metadata object MetadataObject nonExistentObject = NameIdentifierUtil.toMetadataObject( @@ -681,6 +827,13 @@ public class TestTagManager { NameIdentifierUtil.toMetadataObject( NameIdentifierUtil.ofColumn(METALAKE, CATALOG, SCHEMA, TABLE, COLUMN), Entity.EntityType.COLUMN); + MetadataObject viewObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofView(METALAKE, CATALOG, SCHEMA, VIEW), Entity.EntityType.VIEW); + MetadataObject functionObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofFunction(METALAKE, CATALOG, SCHEMA, FUNCTION), + Entity.EntityType.FUNCTION); tagManager.associateTagsForMetadataObject( METALAKE, catalogObject, new String[] {tag1.name(), tag2.name(), tag3.name()}, null); @@ -690,6 +843,10 @@ public class TestTagManager { METALAKE, tableObject, new String[] {tag1.name()}, null); tagManager.associateTagsForMetadataObject( METALAKE, columnObject, new String[] {tag1.name()}, null); + tagManager.associateTagsForMetadataObject( + METALAKE, viewObject, new String[] {tag1.name()}, null); + tagManager.associateTagsForMetadataObject( + METALAKE, functionObject, new String[] {tag1.name()}, null); Tag result = tagManager.getTagForMetadataObject(METALAKE, catalogObject, tag1.name()); Assertions.assertEquals(tag1, result); @@ -706,6 +863,12 @@ public class TestTagManager { Tag result4 = tagManager.getTagForMetadataObject(METALAKE, tableObject, tag1.name()); Assertions.assertEquals(tag1, result4); + Tag result5 = tagManager.getTagForMetadataObject(METALAKE, viewObject, tag1.name()); + Assertions.assertEquals(tag1, result5); + + Tag result6 = tagManager.getTagForMetadataObject(METALAKE, functionObject, tag1.name()); + Assertions.assertEquals(tag1, result6); + // Test get non-existent tag for metadata object Throwable e = Assertions.assertThrows( diff --git a/core/src/test/java/org/apache/gravitino/utils/TestMetadataObjectUtil.java b/core/src/test/java/org/apache/gravitino/utils/TestMetadataObjectUtil.java index 94e93bba66..97a7fef4a4 100644 --- a/core/src/test/java/org/apache/gravitino/utils/TestMetadataObjectUtil.java +++ b/core/src/test/java/org/apache/gravitino/utils/TestMetadataObjectUtil.java @@ -80,6 +80,11 @@ public class TestMetadataObjectUtil { Entity.EntityType.VIEW, MetadataObjectUtil.toEntityType( MetadataObjects.of("catalog.schema", "view", MetadataObject.Type.VIEW))); + + Assertions.assertEquals( + Entity.EntityType.FUNCTION, + MetadataObjectUtil.toEntityType( + MetadataObjects.of("catalog.schema", "function", MetadataObject.Type.FUNCTION))); } @Test @@ -147,6 +152,12 @@ public class TestMetadataObjectUtil { NameIdentifier.of("metalake", "catalog", "schema", "view"), MetadataObjectUtil.toEntityIdent( "metalake", MetadataObjects.of("catalog.schema", "view", MetadataObject.Type.VIEW))); + + Assertions.assertEquals( + NameIdentifier.of("metalake", "catalog", "schema", "function"), + MetadataObjectUtil.toEntityIdent( + "metalake", + MetadataObjects.of("catalog.schema", "function", MetadataObject.Type.FUNCTION))); } @Test diff --git a/docs/open-api/openapi.yaml b/docs/open-api/openapi.yaml index 6b17b53a8a..c5d80ac388 100644 --- a/docs/open-api/openapi.yaml +++ b/docs/open-api/openapi.yaml @@ -594,10 +594,12 @@ components: - "CATALOG" - "SCHEMA" - "TABLE" + - "VIEW" - "COLUMN" - "FILESET" - "TOPIC" - "MODEL" + - "FUNCTION" - "ROLE" metadataObjectFullName: name: metadataObjectFullName @@ -634,4 +636,3 @@ components: KerberosAuth: type: http scheme: negotiate - diff --git a/docs/open-api/tags.yaml b/docs/open-api/tags.yaml index a8b3ac053c..c2e39cd489 100644 --- a/docs/open-api/tags.yaml +++ b/docs/open-api/tags.yaml @@ -222,7 +222,7 @@ paths: tags: - tag summary: Associate tags with metadata object - description: Associate and disassociate tags with metadata object, please be aware that supported metadata objects are CATALOG, SCHEMA, TABLE, FILESET, TOPIC, COLUMN + description: Associate and disassociate tags with metadata object, please be aware that supported metadata objects are CATALOG, SCHEMA, TABLE, VIEW, FILESET, TOPIC, COLUMN, MODEL, FUNCTION operationId: associateTags requestBody: content: @@ -392,9 +392,11 @@ components: - "CATALOG" - "SCHEMA" - "TABLE" + - "VIEW" - "FILESET" - "TOPIC" - "MODEL" + - "FUNCTION" - "COLUMN" diff --git a/server/src/test/java/org/apache/gravitino/server/web/rest/TestMetadataObjectTagOperations.java b/server/src/test/java/org/apache/gravitino/server/web/rest/TestMetadataObjectTagOperations.java index 158324bb4b..e9e7796521 100644 --- a/server/src/test/java/org/apache/gravitino/server/web/rest/TestMetadataObjectTagOperations.java +++ b/server/src/test/java/org/apache/gravitino/server/web/rest/TestMetadataObjectTagOperations.java @@ -108,6 +108,9 @@ public class TestMetadataObjectTagOperations extends BaseOperationsTest { MetadataObjects.parse("object1.object2.object3", MetadataObject.Type.TABLE); MetadataObject column = MetadataObjects.parse("object1.object2.object3.object4", MetadataObject.Type.COLUMN); + MetadataObject view = MetadataObjects.parse("object1.object2.view1", MetadataObject.Type.VIEW); + MetadataObject function = + MetadataObjects.parse("object1.object2.function1", MetadataObject.Type.FUNCTION); Tag[] catalogTagInfos = new Tag[] { @@ -133,6 +136,18 @@ public class TestMetadataObjectTagOperations extends BaseOperationsTest { }; when(tagManager.listTagsInfoForMetadataObject(metalake, column)).thenReturn(columnTagInfos); + Tag[] viewTagInfos = + new Tag[] { + TagEntity.builder().withName("tag9").withId(1L).withAuditInfo(testAuditInfo1).build() + }; + when(tagManager.listTagsInfoForMetadataObject(metalake, view)).thenReturn(viewTagInfos); + + Tag[] functionTagInfos = + new Tag[] { + TagEntity.builder().withName("tag11").withId(1L).withAuditInfo(testAuditInfo1).build() + }; + when(tagManager.listTagsInfoForMetadataObject(metalake, function)).thenReturn(functionTagInfos); + // Test catalog tags Response response = target(basePath(metalake)) @@ -275,6 +290,113 @@ public class TestMetadataObjectTagOperations extends BaseOperationsTest { Assertions.assertTrue(resultNames1.contains("tag3")); Assertions.assertTrue(resultNames1.contains("tag5")); + // Test view tags + Response viewResponse = + target(basePath(metalake)) + .path(view.type().toString()) + .path(view.fullName()) + .path("tags") + .queryParam("details", true) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), viewResponse.getStatus()); + + TagListResponse viewTagListResponse = viewResponse.readEntity(TagListResponse.class); + Assertions.assertEquals(0, viewTagListResponse.getCode()); + Assertions.assertEquals( + schemaTagInfos.length + catalogTagInfos.length + viewTagInfos.length, + viewTagListResponse.getTags().length); + + Map<String, Tag> viewResultTags = + Arrays.stream(viewTagListResponse.getTags()) + .collect(Collectors.toMap(Tag::name, Function.identity())); + + Assertions.assertTrue(viewResultTags.containsKey("tag1")); + Assertions.assertTrue(viewResultTags.containsKey("tag3")); + Assertions.assertTrue(viewResultTags.containsKey("tag9")); + + Assertions.assertTrue(viewResultTags.get("tag1").inherited().get()); + Assertions.assertTrue(viewResultTags.get("tag3").inherited().get()); + Assertions.assertFalse(viewResultTags.get("tag9").inherited().get()); + + Response viewNameResponse = + target(basePath(metalake)) + .path(view.type().toString()) + .path(view.fullName()) + .path("tags") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), viewNameResponse.getStatus()); + + NameListResponse viewNameListResponse = viewNameResponse.readEntity(NameListResponse.class); + Assertions.assertEquals(0, viewNameListResponse.getCode()); + Assertions.assertEquals( + schemaTagInfos.length + catalogTagInfos.length + viewTagInfos.length, + viewNameListResponse.getNames().length); + + Set<String> viewResultNames = Sets.newHashSet(viewNameListResponse.getNames()); + Assertions.assertTrue(viewResultNames.contains("tag1")); + Assertions.assertTrue(viewResultNames.contains("tag3")); + Assertions.assertTrue(viewResultNames.contains("tag9")); + + // Test function tags + Response functionResponse = + target(basePath(metalake)) + .path(function.type().toString()) + .path(function.fullName()) + .path("tags") + .queryParam("details", true) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), functionResponse.getStatus()); + + TagListResponse functionTagListResponse = functionResponse.readEntity(TagListResponse.class); + Assertions.assertEquals(0, functionTagListResponse.getCode()); + Assertions.assertEquals( + schemaTagInfos.length + catalogTagInfos.length + functionTagInfos.length, + functionTagListResponse.getTags().length); + + Map<String, Tag> functionResultTags = + Arrays.stream(functionTagListResponse.getTags()) + .collect(Collectors.toMap(Tag::name, Function.identity())); + + Assertions.assertTrue(functionResultTags.containsKey("tag1")); + Assertions.assertTrue(functionResultTags.containsKey("tag3")); + Assertions.assertTrue(functionResultTags.containsKey("tag11")); + + Assertions.assertTrue(functionResultTags.get("tag1").inherited().get()); + Assertions.assertTrue(functionResultTags.get("tag3").inherited().get()); + Assertions.assertFalse(functionResultTags.get("tag11").inherited().get()); + + Response functionNameResponse = + target(basePath(metalake)) + .path(function.type().toString()) + .path(function.fullName()) + .path("tags") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), functionNameResponse.getStatus()); + + NameListResponse functionNameListResponse = + functionNameResponse.readEntity(NameListResponse.class); + Assertions.assertEquals(0, functionNameListResponse.getCode()); + Assertions.assertEquals( + schemaTagInfos.length + catalogTagInfos.length + functionTagInfos.length, + functionNameListResponse.getNames().length); + + Set<String> functionResultNames = Sets.newHashSet(functionNameListResponse.getNames()); + Assertions.assertTrue(functionResultNames.contains("tag1")); + Assertions.assertTrue(functionResultNames.contains("tag3")); + Assertions.assertTrue(functionResultNames.contains("tag11")); + // Test column tags Response response6 = target(basePath(metalake)) @@ -510,6 +632,17 @@ public class TestMetadataObjectTagOperations extends BaseOperationsTest { MetadataObjects.parse("object1.object2.object3.object4", MetadataObject.Type.COLUMN); when(tagManager.getTagForMetadataObject(metalake, column, "tag4")).thenReturn(tag4); + TagEntity tag5 = + TagEntity.builder().withName("tag5").withId(1L).withAuditInfo(testAuditInfo1).build(); + MetadataObject view = MetadataObjects.parse("object1.object2.view1", MetadataObject.Type.VIEW); + when(tagManager.getTagForMetadataObject(metalake, view, "tag5")).thenReturn(tag5); + + TagEntity tag6 = + TagEntity.builder().withName("tag6").withId(1L).withAuditInfo(testAuditInfo1).build(); + MetadataObject function = + MetadataObjects.parse("object1.object2.function1", MetadataObject.Type.FUNCTION); + when(tagManager.getTagForMetadataObject(metalake, function, "tag6")).thenReturn(tag6); + // Test catalog tag Response response = target(basePath(metalake)) @@ -657,6 +790,92 @@ public class TestMetadataObjectTagOperations extends BaseOperationsTest { Assertions.assertEquals(tag3.comment(), respTag6.comment()); Assertions.assertTrue(respTag6.inherited().get()); + // Test view tag + Response viewResponse = + target(basePath(metalake)) + .path(view.type().toString()) + .path(view.fullName()) + .path("tags") + .path("tag5") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), viewResponse.getStatus()); + + TagResponse viewTagResponse = viewResponse.readEntity(TagResponse.class); + Assertions.assertEquals(0, viewTagResponse.getCode()); + + Tag viewRespTag = viewTagResponse.getTag(); + Assertions.assertEquals(tag5.name(), viewRespTag.name()); + Assertions.assertEquals(tag5.comment(), viewRespTag.comment()); + Assertions.assertFalse(viewRespTag.inherited().get()); + + // Test get view inherited tag + Response viewInheritedResponse = + target(basePath(metalake)) + .path(view.type().toString()) + .path(view.fullName()) + .path("tags") + .path("tag2") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), viewInheritedResponse.getStatus()); + + TagResponse viewInheritedTagResponse = viewInheritedResponse.readEntity(TagResponse.class); + Assertions.assertEquals(0, viewInheritedTagResponse.getCode()); + + Tag viewInheritedRespTag = viewInheritedTagResponse.getTag(); + Assertions.assertEquals(tag2.name(), viewInheritedRespTag.name()); + Assertions.assertEquals(tag2.comment(), viewInheritedRespTag.comment()); + Assertions.assertTrue(viewInheritedRespTag.inherited().get()); + + // Test function tag + Response functionResponse = + target(basePath(metalake)) + .path(function.type().toString()) + .path(function.fullName()) + .path("tags") + .path("tag6") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), functionResponse.getStatus()); + + TagResponse functionTagResponse = functionResponse.readEntity(TagResponse.class); + Assertions.assertEquals(0, functionTagResponse.getCode()); + + Tag functionRespTag = functionTagResponse.getTag(); + Assertions.assertEquals(tag6.name(), functionRespTag.name()); + Assertions.assertEquals(tag6.comment(), functionRespTag.comment()); + Assertions.assertFalse(functionRespTag.inherited().get()); + + // Test get function inherited tag + Response functionInheritedResponse = + target(basePath(metalake)) + .path(function.type().toString()) + .path(function.fullName()) + .path("tags") + .path("tag2") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals( + Response.Status.OK.getStatusCode(), functionInheritedResponse.getStatus()); + + TagResponse functionInheritedTagResponse = + functionInheritedResponse.readEntity(TagResponse.class); + Assertions.assertEquals(0, functionInheritedTagResponse.getCode()); + + Tag functionInheritedRespTag = functionInheritedTagResponse.getTag(); + Assertions.assertEquals(tag2.name(), functionInheritedRespTag.name()); + Assertions.assertEquals(tag2.comment(), functionInheritedRespTag.comment()); + Assertions.assertTrue(functionInheritedRespTag.inherited().get()); + // Test catalog tag throw NoSuchTagException Response response7 = target(basePath(metalake)) @@ -758,12 +977,49 @@ public class TestMetadataObjectTagOperations extends BaseOperationsTest { Assertions.assertEquals( TagAlreadyAssociatedException.class.getSimpleName(), errorResponse.getType()); + // Test associate tags for view + MetadataObject view = MetadataObjects.parse("object1.object2.view1", MetadataObject.Type.VIEW); + when(tagManager.associateTagsForMetadataObject(metalake, view, tagsToAdd, tagsToRemove)) + .thenReturn(tagsToAdd); + + Response response3 = + target(basePath(metalake)) + .path(view.type().toString()) + .path(view.fullName()) + .path("tags") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response3.getStatus()); + Assertions.assertArrayEquals( + tagsToAdd, response3.readEntity(NameListResponse.class).getNames()); + + // Test associate tags for function + MetadataObject function = + MetadataObjects.parse("object1.object2.function1", MetadataObject.Type.FUNCTION); + when(tagManager.associateTagsForMetadataObject(metalake, function, tagsToAdd, tagsToRemove)) + .thenReturn(tagsToAdd); + + Response response4 = + target(basePath(metalake)) + .path(function.type().toString()) + .path(function.fullName()) + .path("tags") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response4.getStatus()); + Assertions.assertArrayEquals( + tagsToAdd, response4.readEntity(NameListResponse.class).getNames()); + // Test throw RuntimeException doThrow(new RuntimeException("mock error")) .when(tagManager) .associateTagsForMetadataObject(any(), any(), any(), any()); - Response response3 = + Response response5 = target(basePath(metalake)) .path(catalog.type().toString()) .path(catalog.fullName()) @@ -773,9 +1029,9 @@ public class TestMetadataObjectTagOperations extends BaseOperationsTest { .post(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); Assertions.assertEquals( - Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), response3.getStatus()); + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), response5.getStatus()); - ErrorResponse errorResponse1 = response3.readEntity(ErrorResponse.class); + ErrorResponse errorResponse1 = response5.readEntity(ErrorResponse.class); Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResponse1.getCode()); Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResponse1.getType()); }
