This is an automated email from the ASF dual-hosted git repository.

dengzh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new ef3bd895390 HIVE-28990: Collecting the non-native summary can be slow 
to raise th… (#5843)
ef3bd895390 is described below

commit ef3bd89539068c3247ea20ebd0b4a9ff44fa0fbd
Author: dengzh <[email protected]>
AuthorDate: Sun Jun 8 08:19:21 2025 +0800

    HIVE-28990: Collecting the non-native summary can be slow to raise th… 
(#5843)
---
 .../hadoop/hive/metastore/conf/MetastoreConf.java  |  2 +
 .../metastore/metasummary/MetaSummaryHandler.java  |  6 +--
 .../metasummary/MetadataTableSummary.java          | 12 +-----
 .../hive/metastore/tools/MetaToolObjectStore.java  |  2 +-
 .../metatool/MetaToolTaskMetadataSummary.java      | 47 +++++++++++++++++-----
 5 files changed, 44 insertions(+), 25 deletions(-)

diff --git 
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
 
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index 6f13260bd3a..9b47e1ac939 100644
--- 
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++ 
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -1905,6 +1905,8 @@ public enum ConfVars {
         "Only collect the non-native table's summary that has been 
updated/changed in configured recent days."),
     
METADATA_SUMMARY_MAX_NONNATIVE_TABLES("hive.metatool.summary.maxNonNativeTables",
 "hive.metatool.summary.maxNonNativeTables", "",
         "The maximum non-native tables allowed per table type during 
collecting the summary."),
+    
METADATA_SUMMARY_NONNATIVE_THREADS("hive.metatool.summary.nonnative.threads", 
"hive.metatool.summary.nonnative.threads", 20,
+        "Number of threads to be allocated for MetaToolTaskMetadataSummary for 
collecting the non-native table's summary."),
 
     // These are all values that we put here just for testing
     STR_TEST_ENTRY("test.str", "hive.test.str", "defaultval", "comment"),
diff --git 
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/metasummary/MetaSummaryHandler.java
 
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/metasummary/MetaSummaryHandler.java
index 86549ae672b..6db37c560dc 100644
--- 
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/metasummary/MetaSummaryHandler.java
+++ 
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/metasummary/MetaSummaryHandler.java
@@ -32,7 +32,7 @@ public interface MetaSummaryHandler extends AutoCloseable, 
Configurable {
    * @param formatJson True if the summary is printed as json, false otherwise.
    * @param schema Extra summary fields
    */
-  public void initialize(String catalog,
+  void initialize(String catalog,
       boolean formatJson, MetaSummarySchema schema) throws 
SummaryInitializationException;
 
   /**
@@ -40,12 +40,12 @@ public void initialize(String catalog,
    * @param tableName The input table
    * @param tableSummary The original summary collected from HMS for this 
table, which needs to be updated.
    */
-  public void appendSummary(TableName tableName, MetadataTableSummary 
tableSummary);
+  void appendSummary(TableName tableName, MetadataTableSummary tableSummary);
 
   /**
    * The exception throws from the summary initialization
    */
-  public class SummaryInitializationException extends RuntimeException {
+  class SummaryInitializationException extends RuntimeException {
     public SummaryInitializationException(String message) {
       super(message);
     }
diff --git 
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/metasummary/MetadataTableSummary.java
 
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/metasummary/MetadataTableSummary.java
index 44ccefa71e9..442904bc399 100644
--- 
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/metasummary/MetadataTableSummary.java
+++ 
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/metasummary/MetadataTableSummary.java
@@ -54,26 +54,18 @@ public MetadataTableSummary(String catalogName,
   public MetadataTableSummary() {
   }
 
-  public MetadataTableSummary columnSummary(int columnCount,
+  public void columnSummary(int columnCount,
       int arrayColCount, int structColCount, int mapColCount) {
     this.colCount = columnCount;
     this.arrayColumnCount = arrayColCount;
     this.structColumnCount = structColCount;
     this.mapColumnCount = mapColCount;
-    return this;
   }
 
-  public MetadataTableSummary partitionSummary(int partColCount, int 
partCount) {
-    this.partitionCount = partCount;
-    this.partitionColumnCount = partColCount;
-    return this;
-  }
-
-  public MetadataTableSummary tableFormatSummary(String tblType, String 
compression, String fileFormat) {
+  public void tableFormatSummary(String tblType, String compression, String 
fileFormat) {
     this.tableType = tblType;
     this.compressionType = compression;
     this.fileFormat = fileFormat;
-    return this;
   }
 
   public String getOwner() {
diff --git 
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/MetaToolObjectStore.java
 
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/MetaToolObjectStore.java
index 17b0d6b3642..f99d86895d0 100644
--- 
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/MetaToolObjectStore.java
+++ 
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/MetaToolObjectStore.java
@@ -905,7 +905,7 @@ private void feedBasicStats(MetadataTableSummary summary, 
String key, Object val
 
   public Set<Long> filterTablesForSummary(List<MetadataTableSummary> 
tableSummaries,
       Integer lastUpdatedDays, Integer tablesLimit) throws MetaException {
-    if (tableSummaries == null || tableSummaries.isEmpty()) {
+    if (tableSummaries == null || tableSummaries.isEmpty() || (tablesLimit != 
null && tablesLimit == 0)) {
       return Collections.emptySet();
     }
     Set<Long> tableIds = 
tableSummaries.stream().map(MetadataTableSummary::getTableId).collect(Collectors.toSet());
diff --git 
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/MetaToolTaskMetadataSummary.java
 
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/MetaToolTaskMetadataSummary.java
index ba9d720f608..4edbf5121d0 100644
--- 
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/MetaToolTaskMetadataSummary.java
+++ 
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/MetaToolTaskMetadataSummary.java
@@ -20,6 +20,7 @@
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ArrayListMultimap;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
 import com.google.gson.JsonArray;
@@ -32,7 +33,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
-import java.util.IdentityHashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -169,6 +170,7 @@ String[] validateInput(String[] inputParams) {
   Pair<MetaSummarySchema, List<MetadataTableSummary>> obtainAndFilterSummary() 
throws MetaException {
     Deadline.registerIfNot(taskTimeout);
     boolean isTimerStarted = false;
+    ExecutorService service = null;
     try {
       isTimerStarted = Deadline.startTimer("obtainAndFilterSummary");
       MetaToolObjectStore objectStore = getObjectStore();
@@ -179,29 +181,49 @@ Pair<MetaSummarySchema, List<MetadataTableSummary>> 
obtainAndFilterSummary() thr
       }
       ArrayListMultimap<Class<? extends MetaSummaryHandler>, 
MetadataTableSummary> nonNativeSummaries =
           findNonNativeSummaries(allSummaries);
-      Map<MetadataTableSummary, Void> filteredSummary = new 
IdentityHashMap<>();
+      Set<MetadataTableSummary> filteredSummary = new HashSet<>();
       MetaSummarySchema extraSchema = new MetaSummarySchema();
       for (Class<? extends MetaSummaryHandler> handler : 
nonNativeSummaries.keys()) {
         Configuration conf = getObjectStore().getConf();
+        List<Future<?>> futures = new ArrayList<>();
         try (MetaSummaryHandler summaryHandler = 
JavaUtils.newInstance(handler)) {
           summaryHandler.setConf(conf);
           summaryHandler.initialize(MetaStoreUtils.getDefaultCatalog(conf), 
formatJson, extraSchema);
           List<MetadataTableSummary> tableSummaries = 
nonNativeSummaries.get(handler);
           // Filter those we don't want to collect
           Set<Long> tableIds = 
getObjectStore().filterTablesForSummary(tableSummaries, recentUpdatedDays, 
maxNonNativeTables);
+          if (service == null) {
+            int nThreads = Math.min(MetastoreConf.getIntVar(conf, 
MetastoreConf.ConfVars.METADATA_SUMMARY_NONNATIVE_THREADS),
+                tableIds.size());
+            if (nThreads > 1) {
+              service = Executors.newFixedThreadPool(nThreads,
+                  new 
ThreadFactoryBuilder().setDaemon(true).setNameFormat("MetaToolTaskMetadataSummary
 #%d").build());
+            }
+          }
           for (MetadataTableSummary summary : tableSummaries) {
-            if (tableIds.contains(summary.getTableId())) {
+            if (!tableIds.contains(summary.getTableId())) {
+              filteredSummary.add(summary);
+            } else {
               TableName tableName = new TableName(summary.getCatalogName(),
                   summary.getDbName(), summary.getTblName());
-              summaryHandler.appendSummary(tableName, summary);
-            } else {
-              filteredSummary.put(summary, null);
-            }
-            // If there is an exception while collecting the summary, remove it
-            if (summary.isDropped()) {
-              filteredSummary.put(summary, null);
+              Runnable task = () -> {
+                summaryHandler.appendSummary(tableName, summary);
+                // If there is an exception while collecting the summary, 
remove it
+                if (summary.isDropped()) {
+                  filteredSummary.add(summary);
+                }
+              };
+              if (service != null) {
+                futures.add(service.submit(task));
+              } else {
+                task.run();
+              }
             }
           }
+          // Waiting for the result before closing the MetaSummaryHandler
+          for (Future<?> future : futures) {
+            future.get();
+          }
         } catch (Exception e) {
           System.err.println(ExceptionUtils.getStackTrace(e));
           LOG.warn("Error collecting the summary from handler: " + 
handler.getName(), e);
@@ -210,10 +232,13 @@ Pair<MetaSummarySchema, List<MetadataTableSummary>> 
obtainAndFilterSummary() thr
       // Filter the table summary from the output
       if (!filteredSummary.isEmpty()) {
         allSummaries = allSummaries.stream()
-            .filter(s -> 
!filteredSummary.containsKey(s)).collect(Collectors.toList());
+            .filter(s -> 
!filteredSummary.contains(s)).collect(Collectors.toList());
       }
       return Pair.of(extraSchema, allSummaries);
     } finally {
+      if (service != null) {
+        service.shutdownNow();
+      }
       if (isTimerStarted) {
         Deadline.stopTimer();
       }

Reply via email to