This is an automated email from the ASF dual-hosted git repository.
jshao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new fbdb14f083 [#6744] improvment(core): Batch get metadata objects for
TagMetaService (#6752)
fbdb14f083 is described below
commit fbdb14f0835be9945cf8cda03854e753058eb274
Author: Mini Yu <[email protected]>
AuthorDate: Sat Apr 5 01:40:18 2025 +0800
[#6744] improvment(core): Batch get metadata objects for TagMetaService
(#6752)
### What changes were proposed in this pull request?
Group by metadata object by metadata type and batch get all metadata
with the same type.
### Why are the changes needed?
To return the times to access databases and improve performance.
Fix: #6744
### Does this PR introduce _any_ user-facing change?
N/A
### How was this patch tested?
Test locally and existing tests.
---
.../relational/mapper/TableColumnMapper.java | 3 +
.../mapper/TableColumnSQLProviderFactory.java | 4 +
.../provider/base/TableColumnBaseSQLProvider.java | 33 ++++
.../relational/service/MetadataObjectService.java | 189 +++++++--------------
.../relational/service/RoleMetaService.java | 21 +--
.../storage/relational/service/TagMetaService.java | 40 +++--
6 files changed, 129 insertions(+), 161 deletions(-)
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableColumnMapper.java
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableColumnMapper.java
index 87b38ea482..64859c5921 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableColumnMapper.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableColumnMapper.java
@@ -71,4 +71,7 @@ public interface TableColumnMapper {
@SelectProvider(type = TableColumnSQLProviderFactory.class, method =
"selectColumnPOById")
ColumnPO selectColumnPOById(@Param("columnId") Long columnId);
+
+ @SelectProvider(type = TableColumnSQLProviderFactory.class, method =
"listColumnPOsByColumnIds")
+ List<ColumnPO> listColumnPOsByColumnIds(@Param("columnIds") List<Long>
columnIds);
}
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableColumnSQLProviderFactory.java
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableColumnSQLProviderFactory.java
index 11f0d5419f..a0e7655e2b 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableColumnSQLProviderFactory.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableColumnSQLProviderFactory.java
@@ -90,4 +90,8 @@ public class TableColumnSQLProviderFactory {
public static String selectColumnPOById(@Param("columnId") Long columnId) {
return getProvider().selectColumnPOById(columnId);
}
+
+ public static String listColumnPOsByColumnIds(@Param("columnIds") List<Long>
columnIds) {
+ return getProvider().listColumnPOsByColumnIds(columnIds);
+ }
}
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/TableColumnBaseSQLProvider.java
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/TableColumnBaseSQLProvider.java
index 1ed117b8da..0b59855023 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/TableColumnBaseSQLProvider.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/TableColumnBaseSQLProvider.java
@@ -132,4 +132,37 @@ public class TableColumnBaseSQLProvider {
+ " WHERE column_id = #{columnId} AND deleted_at = 0"
+ " ORDER BY table_version DESC LIMIT 1";
}
+
+ public String listColumnPOsByColumnIds(@Param("columnIds") List<Long>
columnIds) {
+ return "<script>"
+ + " SELECT c.column_id AS columnId, c.column_name AS columnName,"
+ + " c.column_position AS columnPosition, c.metalake_id AS metalakeId,
c.catalog_id AS catalogId,"
+ + " c.schema_id AS schemaId, c.table_id AS tableId,"
+ + " c.table_version AS tableVersion, c.column_type AS columnType,"
+ + " c.column_comment AS columnComment, c.column_nullable AS nullable,"
+ + " c.column_auto_increment AS autoIncrement,"
+ + " c.column_default_value AS defaultValue, c.column_op_type AS
columnOpType,"
+ + " c.deleted_at AS deletedAt, c.audit_info AS auditInfo"
+ + " FROM "
+ + TableColumnMapper.COLUMN_TABLE_NAME
+ + " c "
+ + "JOIN ("
+ + " SELECT column_id, MAX(table_version) AS max_version"
+ + " FROM "
+ + TableColumnMapper.COLUMN_TABLE_NAME
+ + " WHERE column_id IN ("
+ + " <foreach collection='columnIds' item='columnId'
separator=','>"
+ + " #{columnId}"
+ + " </foreach>"
+ + " ) AND deleted_at = 0 GROUP BY column_id"
+ + ") latest "
+ + "ON c.column_id = latest.column_id AND c.table_version =
latest.max_version"
+ + " WHERE c.column_id IN ("
+ + "<foreach collection='columnIds' item='columnId' separator=','>"
+ + "#{columnId}"
+ + "</foreach>"
+ + ") "
+ + " AND c.deleted_at = 0"
+ + "</script>";
+ }
}
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java
b/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java
index 188c06ac23..8ecc218626 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java
@@ -20,17 +20,19 @@ package org.apache.gravitino.storage.relational.service;
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableMap;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.function.Function;
import java.util.stream.Collectors;
-import javax.annotation.Nullable;
import org.apache.gravitino.MetadataObject;
import org.apache.gravitino.storage.relational.mapper.CatalogMetaMapper;
import org.apache.gravitino.storage.relational.mapper.FilesetMetaMapper;
import org.apache.gravitino.storage.relational.mapper.MetalakeMetaMapper;
import org.apache.gravitino.storage.relational.mapper.ModelMetaMapper;
import org.apache.gravitino.storage.relational.mapper.SchemaMetaMapper;
+import org.apache.gravitino.storage.relational.mapper.TableColumnMapper;
import org.apache.gravitino.storage.relational.mapper.TableMetaMapper;
import org.apache.gravitino.storage.relational.mapper.TopicMetaMapper;
import org.apache.gravitino.storage.relational.po.CatalogPO;
@@ -57,6 +59,18 @@ public class MetadataObjectService {
private static final Logger LOG =
LoggerFactory.getLogger(MetadataObjectService.class);
+ static final Map<MetadataObject.Type, Function<List<Long>, Map<Long,
String>>>
+ TYPE_TO_FULLNAME_FUNCTION_MAP =
+ ImmutableMap.of(
+ MetadataObject.Type.METALAKE,
MetadataObjectService::getMetalakeObjectsFullName,
+ MetadataObject.Type.CATALOG,
MetadataObjectService::getCatalogObjectsFullName,
+ MetadataObject.Type.SCHEMA,
MetadataObjectService::getSchemaObjectsFullName,
+ MetadataObject.Type.TABLE,
MetadataObjectService::getTableObjectsFullName,
+ MetadataObject.Type.FILESET,
MetadataObjectService::getFilesetObjectsFullName,
+ MetadataObject.Type.MODEL,
MetadataObjectService::getModelObjectsFullName,
+ MetadataObject.Type.TOPIC,
MetadataObjectService::getTopicObjectsFullName,
+ MetadataObject.Type.COLUMN,
MetadataObjectService::getColumnObjectsFullName);
+
private MetadataObjectService() {}
public static long getMetadataObjectId(
@@ -105,134 +119,6 @@ public class MetadataObjectService {
throw new IllegalArgumentException(String.format("Doesn't support the type
%s", type));
}
- // Metadata object may be null because the metadata object can be deleted
- // asynchronously.
- @Nullable
- public static String getMetadataObjectFullName(String type, long
metadataObjectId) {
- MetadataObject.Type metadataType = MetadataObject.Type.valueOf(type);
- String fullName = null;
- long objectId = metadataObjectId;
-
- do {
- switch (metadataType) {
- case METALAKE:
- MetalakePO metalakePO =
MetalakeMetaService.getInstance().getMetalakePOById(objectId);
- if (metalakePO != null) {
- fullName = metalakePO.getMetalakeName();
- metadataType = null;
- } else {
- return null;
- }
- break;
-
- case CATALOG:
- CatalogPO catalogPO =
CatalogMetaService.getInstance().getCatalogPOById(objectId);
- if (catalogPO != null) {
- fullName =
- fullName != null
- ? DOT_JOINER.join(catalogPO.getCatalogName(), fullName)
- : catalogPO.getCatalogName();
- metadataType = null;
- } else {
- return null;
- }
- break;
-
- case SCHEMA:
- SchemaPO schemaPO =
SchemaMetaService.getInstance().getSchemaPOById(objectId);
- if (schemaPO != null) {
- fullName =
- fullName != null
- ? DOT_JOINER.join(schemaPO.getSchemaName(), fullName)
- : schemaPO.getSchemaName();
- objectId = schemaPO.getCatalogId();
- metadataType = MetadataObject.Type.CATALOG;
- } else {
- return null;
- }
- break;
-
- case TABLE:
- TablePO tablePO =
TableMetaService.getInstance().getTablePOById(objectId);
- // if fullName is null:
- // fullName = catalogPO.getSchemaName(),schemaPO.getTableName()
- if (tablePO != null) {
- fullName =
- fullName != null
- ? DOT_JOINER.join(tablePO.getTableName(), fullName)
- : tablePO.getTableName();
- objectId = tablePO.getSchemaId();
- metadataType = MetadataObject.Type.SCHEMA;
- } else {
- return null;
- }
- break;
-
- case TOPIC:
- TopicPO topicPO =
TopicMetaService.getInstance().getTopicPOById(objectId);
- if (topicPO != null) {
- fullName =
- fullName != null
- ? DOT_JOINER.join(topicPO.getTopicName(), fullName)
- : topicPO.getTopicName();
- objectId = topicPO.getSchemaId();
- metadataType = MetadataObject.Type.SCHEMA;
- } else {
- return null;
- }
- break;
-
- case FILESET:
- FilesetPO filesetPO =
FilesetMetaService.getInstance().getFilesetPOById(objectId);
- if (filesetPO != null) {
- fullName =
- fullName != null
- ? DOT_JOINER.join(filesetPO.getFilesetName(), fullName)
- : filesetPO.getFilesetName();
- objectId = filesetPO.getSchemaId();
- metadataType = MetadataObject.Type.SCHEMA;
- } else {
- return null;
- }
- break;
-
- case MODEL:
- ModelPO modelPO =
ModelMetaService.getInstance().getModelPOById(objectId);
- if (modelPO != null) {
- fullName =
- fullName != null
- ? DOT_JOINER.join(modelPO.getModelName(), fullName)
- : modelPO.getModelName();
- objectId = modelPO.getSchemaId();
- metadataType = MetadataObject.Type.SCHEMA;
- } else {
- return null;
- }
- break;
-
- case COLUMN:
- ColumnPO columnPO =
TableColumnMetaService.getInstance().getColumnPOById(objectId);
- if (columnPO != null) {
- fullName =
- fullName != null
- ? DOT_JOINER.join(columnPO.getColumnName(), fullName)
- : columnPO.getColumnName();
- objectId = columnPO.getTableId();
- metadataType = MetadataObject.Type.TABLE;
- } else {
- return null;
- }
- break;
-
- default:
- throw new IllegalArgumentException(
- String.format("Doesn't support the type %s", metadataType));
- }
- } while (metadataType != null);
-
- return fullName;
- }
-
/**
* Retrieves a map of Metalake object IDs to their full names.
*
@@ -389,6 +275,51 @@ public class MetadataObjectService {
return tableIdAndNameMap;
}
+ /**
+ * Retrieves a map of column object IDs to their full names.
+ *
+ * @param columnsIds A list of column object IDs to fetch names for.
+ * @return A Map where the key is the column ID and the value is the column
full name. The map may
+ * contain null values for the names if its parent object is deleted.
Returns an empty map if
+ * no column objects are found for the given IDs. {@code @example} value
of table full name:
+ * "catalog1.schema1.table1.column1"
+ */
+ public static Map<Long, String> getColumnObjectsFullName(List<Long>
columnsIds) {
+ List<ColumnPO> columnPOs =
+ SessionUtils.getWithoutCommit(
+ TableColumnMapper.class, mapper ->
mapper.listColumnPOsByColumnIds(columnsIds));
+
+ if (columnPOs == null || columnPOs.isEmpty()) {
+ return new HashMap<>();
+ }
+
+ List<Long> tableIds =
columnPOs.stream().map(ColumnPO::getTableId).collect(Collectors.toList());
+ Map<Long, String> tableIdAndNameMap = getTableObjectsFullName(tableIds);
+
+ HashMap<Long, String> columnIdAndNameMap = new HashMap<>();
+
+ columnPOs.forEach(
+ columnPO -> {
+ // since the table can be deleted, we need to check the null value,
+ // and when the table is deleted, we will set fullName of column to
+ // null
+ String tableName =
tableIdAndNameMap.getOrDefault(columnPO.getTableId(), null);
+ if (tableName == null) {
+ LOG.warn(
+ "The table '{}' of column '{}' may be deleted",
+ columnPO.getTableId(),
+ columnPO.getColumnId());
+ columnIdAndNameMap.put(columnPO.getColumnId(), null);
+ return;
+ }
+
+ String fullName = DOT_JOINER.join(tableName,
columnPO.getColumnName());
+ columnIdAndNameMap.put(columnPO.getColumnId(), fullName);
+ });
+
+ return columnIdAndNameMap;
+ }
+
/**
* Retrieves a map of Topic object IDs to their full names.
*
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java
b/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java
index 59ec4dea8c..58e4a6e7fc 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java
@@ -19,7 +19,6 @@
package org.apache.gravitino.storage.relational.service;
import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import java.io.IOException;
@@ -364,28 +363,10 @@ public class RoleMetaService {
.map(SecurableObjectPO::getMetadataObjectId)
.collect(Collectors.toList());
- Map<MetadataObject.Type, Function<List<Long>, Map<Long, String>>>
- objectFullNameGetterFnMap =
- ImmutableMap.of(
- MetadataObject.Type.METALAKE,
- MetadataObjectService::getMetalakeObjectsFullName,
- MetadataObject.Type.CATALOG,
- MetadataObjectService::getCatalogObjectsFullName,
- MetadataObject.Type.SCHEMA,
- MetadataObjectService::getSchemaObjectsFullName,
- MetadataObject.Type.TABLE,
- MetadataObjectService::getTableObjectsFullName,
- MetadataObject.Type.FILESET,
- MetadataObjectService::getFilesetObjectsFullName,
- MetadataObject.Type.MODEL,
- MetadataObjectService::getModelObjectsFullName,
- MetadataObject.Type.TOPIC,
- MetadataObjectService::getTopicObjectsFullName);
-
// dynamically calling getter function based on type
Map<Long, String> objectIdAndNameMap =
Optional.of(MetadataObject.Type.valueOf(type))
- .map(objectFullNameGetterFnMap::get)
+
.map(MetadataObjectService.TYPE_TO_FULLNAME_FUNCTION_MAP::get)
.map(getter -> getter.apply(objectIds))
.orElseThrow(
() ->
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/service/TagMetaService.java
b/core/src/main/java/org/apache/gravitino/storage/relational/service/TagMetaService.java
index 5863877ae7..3b2c6d42e2 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/service/TagMetaService.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/service/TagMetaService.java
@@ -24,6 +24,7 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import java.util.Map;
import java.util.Objects;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -230,19 +231,34 @@ public class TagMetaService {
mapper.listTagMetadataObjectRelsByMetalakeAndTagName(metalakeName, tagName));
List<MetadataObject> metadataObjects = Lists.newArrayList();
- for (TagMetadataObjectRelPO po : tagMetadataObjectRelPOs) {
- String fullName =
- MetadataObjectService.getMetadataObjectFullName(
- po.getMetadataObjectType(), po.getMetadataObjectId());
-
- // Metadata object may be deleted asynchronously when we query the
name, so it will return
- // null. We should skip this metadata object.
- if (fullName == null) {
- continue;
+ Map<String, List<TagMetadataObjectRelPO>> tagMetadataObjectRelPOsByType =
+ tagMetadataObjectRelPOs.stream()
+
.collect(Collectors.groupingBy(TagMetadataObjectRelPO::getMetadataObjectType));
+
+ for (Map.Entry<String, List<TagMetadataObjectRelPO>> entry :
+ tagMetadataObjectRelPOsByType.entrySet()) {
+ String metadataObjectType = entry.getKey();
+ List<TagMetadataObjectRelPO> rels = entry.getValue();
+
+ List<Long> metadataObjectIds =
+ rels.stream()
+ .map(TagMetadataObjectRelPO::getMetadataObjectId)
+ .collect(Collectors.toList());
+ Map<Long, String> metadataObjectNames =
+ MetadataObjectService.TYPE_TO_FULLNAME_FUNCTION_MAP
+ .get(MetadataObject.Type.valueOf(metadataObjectType))
+ .apply(metadataObjectIds);
+
+ for (Map.Entry<Long, String> metadataObjectName :
metadataObjectNames.entrySet()) {
+ String fullName = metadataObjectName.getValue();
+
+ // Metadata object may be deleted asynchronously when we query the
name, so it will
+ // return null, we should skip this metadata object.
+ if (fullName != null) {
+ metadataObjects.add(
+ MetadataObjects.parse(fullName,
MetadataObject.Type.valueOf(metadataObjectType)));
+ }
}
-
- MetadataObject.Type type =
MetadataObject.Type.valueOf(po.getMetadataObjectType());
- metadataObjects.add(MetadataObjects.parse(fullName, type));
}
return metadataObjects;
