This is an automated email from the ASF dual-hosted git repository.

dengzh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new 6e261a32657 HIVE-28545: Remove global lock on HMSHandler.getMSForConf 
which is prone to deadlock (#5477) (Zhihua Deng, reviewed by Denys Kuzmenko)
6e261a32657 is described below

commit 6e261a32657d36185654dd05af7215fe33123878
Author: dengzh <[email protected]>
AuthorDate: Tue Oct 1 10:13:47 2024 +0800

    HIVE-28545: Remove global lock on HMSHandler.getMSForConf which is prone to 
deadlock (#5477) (Zhihua Deng, reviewed by Denys Kuzmenko)
---
 .../apache/hadoop/hive/metastore/HMSHandler.java   |   2 +-
 .../hive/metastore/PersistenceManagerProvider.java |   4 +-
 .../hive/metastore/TestMetaStoreDeadlock.java      | 121 +++++++++++++++++++++
 3 files changed, 124 insertions(+), 3 deletions(-)

diff --git 
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
 
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
index 37450f68c1a..2c1e53b11d6 100644
--- 
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
+++ 
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
@@ -535,7 +535,7 @@ public class HMSHandler extends FacebookBase implements 
IHMSHandler {
     return getMSForConf(conf);
   }
 
-  public static synchronized RawStore getMSForConf(Configuration conf) throws 
MetaException {
+  public static RawStore getMSForConf(Configuration conf) throws MetaException 
{
     RawStore ms = getRawStore();
     if (ms == null) {
       ms = newRawStoreForConf(conf);
diff --git 
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PersistenceManagerProvider.java
 
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PersistenceManagerProvider.java
index f286dc850ed..723ece51a36 100644
--- 
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PersistenceManagerProvider.java
+++ 
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PersistenceManagerProvider.java
@@ -308,8 +308,8 @@ public class PersistenceManagerProvider {
         // The secondary connection factory is used for schema generation, and 
for value generation operations.
         // We should use a different pool for the secondary connection factory 
to avoid resource starvation.
         // Since DataNucleus uses locks for schema generation and value 
generation, 2 connections should be sufficient.
-        configurator.resetName("objectstore-secondary");
-        DataSource ds2 = forCompactor ? ds : dsp.create(conf, /* maxPoolSize 
*/ 2);
+        configurator.resetName(sourceName + "-secondary");
+        DataSource ds2 = dsp.create(conf, /* maxPoolSize */ 2);
         dsProp.put(PropertyNames.PROPERTY_CONNECTION_FACTORY, ds);
         dsProp.put(PropertyNames.PROPERTY_CONNECTION_FACTORY2, ds2);
         dsProp.put(ConfVars.MANAGER_FACTORY_CLASS.getVarname(),
diff --git 
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreDeadlock.java
 
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreDeadlock.java
new file mode 100644
index 00000000000..60d7d8d6ba1
--- /dev/null
+++ 
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreDeadlock.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest;
+import org.apache.hadoop.hive.metastore.api.GetPartitionsByNamesRequest;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.PartitionsRequest;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder;
+import org.apache.hadoop.hive.metastore.client.builder.PartitionBuilder;
+import org.apache.hadoop.hive.metastore.client.builder.TableBuilder;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge;
+import org.apache.thrift.TException;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(MetastoreUnitTest.class)
+public class TestMetaStoreDeadlock {
+  private Configuration conf;
+  private static int POOL_SIZE = 3;
+  private static CountDownLatch LATCH1 = new CountDownLatch(POOL_SIZE + 1);
+  private static CountDownLatch LATCH2 = new CountDownLatch(POOL_SIZE);
+
+  @Before
+  public void setUp() throws Exception {
+    conf = MetastoreConf.newMetastoreConf();
+    MetastoreConf.setLongVar(conf, 
MetastoreConf.ConfVars.CONNECTION_POOLING_MAX_CONNECTIONS, POOL_SIZE);
+    MetastoreConf.setVar(conf, 
MetastoreConf.ConfVars.METASTORE_METADATA_TRANSFORMER_CLASS, 
SleepOnGetPartitions.class.getName());
+    MetastoreConf.setTimeVar(conf, 
MetastoreConf.ConfVars.CLIENT_SOCKET_TIMEOUT, 1, TimeUnit.HOURS);
+    conf.setLong("hikaricp.connectionTimeout", 3600000);
+    
MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge(), 
conf);
+  }
+
+  @Test(timeout = 60000)
+  public void testLockContention() throws Exception {
+    String dbName = "_test_deadlock_";
+    String tableName1 = "tbl1";
+    try (HiveMetaStoreClient msc = new HiveMetaStoreClient(conf)) {
+      new DatabaseBuilder().setName(dbName).create(msc, conf);
+      new 
TableBuilder().setDbName(dbName).setTableName(tableName1).addCol("a", 
"string").addPartCol("dt", "string")
+          .create(msc, conf);
+      Table table1 = msc.getTable(dbName, tableName1);
+      new 
PartitionBuilder().inTable(table1).addValue("2024-09-29").addToTable(msc, conf);
+    }
+    GetPartitionsByNamesRequest request = new 
GetPartitionsByNamesRequest(dbName, tableName1);
+    request.setNames(Arrays.asList("dt=2024-09-28"));
+    request.setProcessorCapabilities(Arrays.asList("HIVEFULLACIDWRITE", 
"HIVEFULLACIDREAD", "HIVEMANAGEDINSERTWRITE"));
+    Thread[] holdConnThreads = new Thread[POOL_SIZE];
+    for (int i = 0; i < POOL_SIZE; i++) {
+      holdConnThreads[i] = new Thread(() -> {
+        try (HiveMetaStoreClient client = new HiveMetaStoreClient(conf)) {
+          LATCH1.countDown();
+          client.getPartitionsByNames(request);
+        } catch (TException e) {
+          throw new RuntimeException(e);
+        }
+      });
+      holdConnThreads[i].start();
+    }
+    LATCH2.await();
+    Thread holdMS = new Thread(() -> {
+      try (HiveMetaStoreClient client = new HiveMetaStoreClient(conf)) {
+        client.getPartitionsRequest(new PartitionsRequest(dbName, 
tableName1)).getPartitions();
+      } catch (TException e) {
+        throw new RuntimeException(e);
+      }
+    });
+    holdMS.start();
+    Thread.sleep(5000);
+    LATCH1.countDown();
+    Assert.assertEquals(0, LATCH1.getCount());
+    // this thread will be hanging on as there is no available connection 
until connection timeout
+    holdMS.join();
+  }
+
+  public static class SleepOnGetPartitions extends MetastoreDefaultTransformer 
{
+
+    public SleepOnGetPartitions(IHMSHandler handler) throws HiveMetaException {
+      super(handler);
+    }
+    @Override
+    public List<Partition> transformPartitions(List<Partition> objects,
+        Table table, List<String> processorCapabilities, String processorId) 
throws MetaException {
+      try {
+        LATCH2.countDown();
+        LATCH1.await();
+      } catch (InterruptedException e) {
+        throw new RuntimeException(e);
+      }
+      return super.transformPartitions(objects, table, processorCapabilities, 
processorId);
+    }
+  }
+
+}

Reply via email to