This is an automated email from the ASF dual-hosted git repository.
dengzh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new ef3bd895390 HIVE-28990: Collecting the non-native summary can be slow
to raise th… (#5843)
ef3bd895390 is described below
commit ef3bd89539068c3247ea20ebd0b4a9ff44fa0fbd
Author: dengzh <[email protected]>
AuthorDate: Sun Jun 8 08:19:21 2025 +0800
HIVE-28990: Collecting the non-native summary can be slow to raise th…
(#5843)
---
.../hadoop/hive/metastore/conf/MetastoreConf.java | 2 +
.../metastore/metasummary/MetaSummaryHandler.java | 6 +--
.../metasummary/MetadataTableSummary.java | 12 +-----
.../hive/metastore/tools/MetaToolObjectStore.java | 2 +-
.../metatool/MetaToolTaskMetadataSummary.java | 47 +++++++++++++++++-----
5 files changed, 44 insertions(+), 25 deletions(-)
diff --git
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index 6f13260bd3a..9b47e1ac939 100644
---
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -1905,6 +1905,8 @@ public enum ConfVars {
"Only collect the non-native table's summary that has been
updated/changed in configured recent days."),
METADATA_SUMMARY_MAX_NONNATIVE_TABLES("hive.metatool.summary.maxNonNativeTables",
"hive.metatool.summary.maxNonNativeTables", "",
"The maximum non-native tables allowed per table type during
collecting the summary."),
+
METADATA_SUMMARY_NONNATIVE_THREADS("hive.metatool.summary.nonnative.threads",
"hive.metatool.summary.nonnative.threads", 20,
+ "Number of threads to be allocated for MetaToolTaskMetadataSummary for
collecting the non-native table's summary."),
// These are all values that we put here just for testing
STR_TEST_ENTRY("test.str", "hive.test.str", "defaultval", "comment"),
diff --git
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/metasummary/MetaSummaryHandler.java
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/metasummary/MetaSummaryHandler.java
index 86549ae672b..6db37c560dc 100644
---
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/metasummary/MetaSummaryHandler.java
+++
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/metasummary/MetaSummaryHandler.java
@@ -32,7 +32,7 @@ public interface MetaSummaryHandler extends AutoCloseable,
Configurable {
* @param formatJson True if the summary is printed as json, false otherwise.
* @param schema Extra summary fields
*/
- public void initialize(String catalog,
+ void initialize(String catalog,
boolean formatJson, MetaSummarySchema schema) throws
SummaryInitializationException;
/**
@@ -40,12 +40,12 @@ public void initialize(String catalog,
* @param tableName The input table
* @param tableSummary The original summary collected from HMS for this
table, which needs to be updated.
*/
- public void appendSummary(TableName tableName, MetadataTableSummary
tableSummary);
+ void appendSummary(TableName tableName, MetadataTableSummary tableSummary);
/**
* The exception throws from the summary initialization
*/
- public class SummaryInitializationException extends RuntimeException {
+ class SummaryInitializationException extends RuntimeException {
public SummaryInitializationException(String message) {
super(message);
}
diff --git
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/metasummary/MetadataTableSummary.java
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/metasummary/MetadataTableSummary.java
index 44ccefa71e9..442904bc399 100644
---
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/metasummary/MetadataTableSummary.java
+++
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/metasummary/MetadataTableSummary.java
@@ -54,26 +54,18 @@ public MetadataTableSummary(String catalogName,
public MetadataTableSummary() {
}
- public MetadataTableSummary columnSummary(int columnCount,
+ public void columnSummary(int columnCount,
int arrayColCount, int structColCount, int mapColCount) {
this.colCount = columnCount;
this.arrayColumnCount = arrayColCount;
this.structColumnCount = structColCount;
this.mapColumnCount = mapColCount;
- return this;
}
- public MetadataTableSummary partitionSummary(int partColCount, int
partCount) {
- this.partitionCount = partCount;
- this.partitionColumnCount = partColCount;
- return this;
- }
-
- public MetadataTableSummary tableFormatSummary(String tblType, String
compression, String fileFormat) {
+ public void tableFormatSummary(String tblType, String compression, String
fileFormat) {
this.tableType = tblType;
this.compressionType = compression;
this.fileFormat = fileFormat;
- return this;
}
public String getOwner() {
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/MetaToolObjectStore.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/MetaToolObjectStore.java
index 17b0d6b3642..f99d86895d0 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/MetaToolObjectStore.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/MetaToolObjectStore.java
@@ -905,7 +905,7 @@ private void feedBasicStats(MetadataTableSummary summary,
String key, Object val
public Set<Long> filterTablesForSummary(List<MetadataTableSummary>
tableSummaries,
Integer lastUpdatedDays, Integer tablesLimit) throws MetaException {
- if (tableSummaries == null || tableSummaries.isEmpty()) {
+ if (tableSummaries == null || tableSummaries.isEmpty() || (tablesLimit !=
null && tablesLimit == 0)) {
return Collections.emptySet();
}
Set<Long> tableIds =
tableSummaries.stream().map(MetadataTableSummary::getTableId).collect(Collectors.toSet());
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/MetaToolTaskMetadataSummary.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/MetaToolTaskMetadataSummary.java
index ba9d720f608..4edbf5121d0 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/MetaToolTaskMetadataSummary.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/MetaToolTaskMetadataSummary.java
@@ -20,6 +20,7 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ArrayListMultimap;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonArray;
@@ -32,7 +33,7 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
-import java.util.IdentityHashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -169,6 +170,7 @@ String[] validateInput(String[] inputParams) {
Pair<MetaSummarySchema, List<MetadataTableSummary>> obtainAndFilterSummary()
throws MetaException {
Deadline.registerIfNot(taskTimeout);
boolean isTimerStarted = false;
+ ExecutorService service = null;
try {
isTimerStarted = Deadline.startTimer("obtainAndFilterSummary");
MetaToolObjectStore objectStore = getObjectStore();
@@ -179,29 +181,49 @@ Pair<MetaSummarySchema, List<MetadataTableSummary>>
obtainAndFilterSummary() thr
}
ArrayListMultimap<Class<? extends MetaSummaryHandler>,
MetadataTableSummary> nonNativeSummaries =
findNonNativeSummaries(allSummaries);
- Map<MetadataTableSummary, Void> filteredSummary = new
IdentityHashMap<>();
+ Set<MetadataTableSummary> filteredSummary = new HashSet<>();
MetaSummarySchema extraSchema = new MetaSummarySchema();
for (Class<? extends MetaSummaryHandler> handler :
nonNativeSummaries.keys()) {
Configuration conf = getObjectStore().getConf();
+ List<Future<?>> futures = new ArrayList<>();
try (MetaSummaryHandler summaryHandler =
JavaUtils.newInstance(handler)) {
summaryHandler.setConf(conf);
summaryHandler.initialize(MetaStoreUtils.getDefaultCatalog(conf),
formatJson, extraSchema);
List<MetadataTableSummary> tableSummaries =
nonNativeSummaries.get(handler);
// Filter those we don't want to collect
Set<Long> tableIds =
getObjectStore().filterTablesForSummary(tableSummaries, recentUpdatedDays,
maxNonNativeTables);
+ if (service == null) {
+ int nThreads = Math.min(MetastoreConf.getIntVar(conf,
MetastoreConf.ConfVars.METADATA_SUMMARY_NONNATIVE_THREADS),
+ tableIds.size());
+ if (nThreads > 1) {
+ service = Executors.newFixedThreadPool(nThreads,
+ new
ThreadFactoryBuilder().setDaemon(true).setNameFormat("MetaToolTaskMetadataSummary
#%d").build());
+ }
+ }
for (MetadataTableSummary summary : tableSummaries) {
- if (tableIds.contains(summary.getTableId())) {
+ if (!tableIds.contains(summary.getTableId())) {
+ filteredSummary.add(summary);
+ } else {
TableName tableName = new TableName(summary.getCatalogName(),
summary.getDbName(), summary.getTblName());
- summaryHandler.appendSummary(tableName, summary);
- } else {
- filteredSummary.put(summary, null);
- }
- // If there is an exception while collecting the summary, remove it
- if (summary.isDropped()) {
- filteredSummary.put(summary, null);
+ Runnable task = () -> {
+ summaryHandler.appendSummary(tableName, summary);
+ // If there is an exception while collecting the summary,
remove it
+ if (summary.isDropped()) {
+ filteredSummary.add(summary);
+ }
+ };
+ if (service != null) {
+ futures.add(service.submit(task));
+ } else {
+ task.run();
+ }
}
}
+ // Waiting for the result before closing the MetaSummaryHandler
+ for (Future<?> future : futures) {
+ future.get();
+ }
} catch (Exception e) {
System.err.println(ExceptionUtils.getStackTrace(e));
LOG.warn("Error collecting the summary from handler: " +
handler.getName(), e);
@@ -210,10 +232,13 @@ Pair<MetaSummarySchema, List<MetadataTableSummary>>
obtainAndFilterSummary() thr
// Filter the table summary from the output
if (!filteredSummary.isEmpty()) {
allSummaries = allSummaries.stream()
- .filter(s ->
!filteredSummary.containsKey(s)).collect(Collectors.toList());
+ .filter(s ->
!filteredSummary.contains(s)).collect(Collectors.toList());
}
return Pair.of(extraSchema, allSummaries);
} finally {
+ if (service != null) {
+ service.shutdownNow();
+ }
if (isTimerStarted) {
Deadline.stopTimer();
}