[hive] branch master updated: HIVE-15956: StackOverflowError when drop lots of partitions (Denys Kuzmenko via Zoltan Haindrich)

[kgyrtkirk]

This is an automated email from the ASF dual-hosted git repository.

kgyrtkirk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new 0505596  HIVE-15956: StackOverflowError when drop lots of partitions 
(Denys Kuzmenko via Zoltan Haindrich)
0505596 is described below

commit 0505596bb122eec537b39170700ca187f5604503
Author: Denys Kuzmenko <dkuzme...@cloudera.com>
AuthorDate: Wed Sep 11 11:39:44 2019 +0200

    HIVE-15956: StackOverflowError when drop lots of partitions (Denys Kuzmenko 
via Zoltan Haindrich)
    
    Signed-off-by: Zoltan Haindrich <k...@rxd.hu>
---
 .../apache/hadoop/hive/metastore/ObjectStore.java  | 71 ++++++++++++++--------
 1 file changed, 45 insertions(+), 26 deletions(-)

diff --git 
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
 
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index f04553f..b40d816 100644
--- 
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ 
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -2563,20 +2563,32 @@ public class ObjectStore implements RawStore, 
Configurable {
   private void dropPartitionsViaJdo(String catName, String dbName, String 
tblName,
       List<String> partNames) throws MetaException {
     boolean success = false;
+
+    if (partNames.isEmpty()) {
+      return;
+    }
     openTransaction();
+
     try {
-      // Delete all things.
-      dropPartitionGrantsNoTxn(catName, dbName, tblName, partNames);
-      dropPartitionAllColumnGrantsNoTxn(catName, dbName, tblName, partNames);
-      dropPartitionColumnStatisticsNoTxn(catName, dbName, tblName, partNames);
+      Batchable.runBatched(batchSize, partNames, new Batchable<String, Void>() 
{
+        @Override
+        public List<Void> run(List<String> input) throws MetaException {
+          // Delete all things.
+          dropPartitionGrantsNoTxn(catName, dbName, tblName, input);
+          dropPartitionAllColumnGrantsNoTxn(catName, dbName, tblName, input);
+          dropPartitionColumnStatisticsNoTxn(catName, dbName, tblName, input);
+
+          // CDs are reused; go try partition SDs, detach all CDs from SDs, 
then remove unused CDs.
+          for (MColumnDescriptor mcd : detachCdsFromSdsNoTxn(catName, dbName, 
tblName, input)) {
+            removeUnusedColumnDescriptor(mcd);
+          }
+          dropPartitionsNoTxn(catName, dbName, tblName, input);
+          return Collections.emptyList();
+        }
+      });
 
-      // CDs are reused; go try partition SDs, detach all CDs from SDs, then 
remove unused CDs.
-      for (MColumnDescriptor mcd : detachCdsFromSdsNoTxn(catName, dbName, 
tblName, partNames)) {
-        removeUnusedColumnDescriptor(mcd);
-      }
-      dropPartitionsNoTxn(catName, dbName, tblName, partNames);
       if (!(success = commitTransaction())) {
-        throw new MetaException("Failed to drop partitions"); // Should not 
happen?
+        throw new MetaException("Failed to drop partitions");
       }
     } finally {
       if (!success) {
@@ -3455,24 +3467,31 @@ public class ObjectStore implements RawStore, 
Configurable {
    * @param partNames Partition names to get the objects for.
    * @return Resulting partitions.
    */
-  private List<Partition> getPartitionsViaOrmFilter(String catName,
-      String dbName, String tblName, List<String> partNames) throws 
MetaException {
+  private List<Partition> getPartitionsViaOrmFilter(String catName, String 
dbName, String tblName,
+      List<String> partNames) throws MetaException {
+
     if (partNames.isEmpty()) {
-      return new ArrayList<>();
-    }
-    ObjectPair<Query, Map<String, String>> queryWithParams =
-        getPartQueryWithParams(catName, dbName, tblName, partNames);
-    Query query = queryWithParams.getFirst();
-    query.setResultClass(MPartition.class);
-    query.setClass(MPartition.class);
-    query.setOrdering("partitionName ascending");
-    @SuppressWarnings("unchecked")
-    List<MPartition> mparts = 
(List<MPartition>)query.executeWithMap(queryWithParams.getSecond());
-    List<Partition> partitions = convertToParts(catName, dbName, tblName, 
mparts);
-    if (query != null) {
-      query.closeAll();
+      return Collections.emptyList();
     }
-    return partitions;
+    return Batchable.runBatched(batchSize, partNames, new Batchable<String, 
Partition>() {
+      @Override
+      public List<Partition> run(List<String> input) throws MetaException {
+        ObjectPair<Query, Map<String, String>> queryWithParams =
+            getPartQueryWithParams(catName, dbName, tblName, input);
+
+        Query query = queryWithParams.getFirst();
+        query.setResultClass(MPartition.class);
+        query.setClass(MPartition.class);
+        query.setOrdering("partitionName ascending");
+
+        @SuppressWarnings("unchecked")
+        List<MPartition> mparts = (List<MPartition>) 
query.executeWithMap(queryWithParams.getSecond());
+        List<Partition> partitions = convertToParts(catName, dbName, tblName, 
mparts);
+        query.closeAll();
+
+        return partitions;
+      }
+    });
   }
 
   private void dropPartitionsNoTxn(String catName, String dbName, String 
tblName, List<String> partNames) {