This is an automated email from the ASF dual-hosted git repository.
zabetak pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new a0465a9aa08 HIVE-28900: Make the objectstore-secondary connection pool
size configurable (#5764)
a0465a9aa08 is described below
commit a0465a9aa08e550f4cc497e929b7b2fe8979d615
Author: Miklos Szurap <[email protected]>
AuthorDate: Thu Apr 17 13:40:34 2025 +0200
HIVE-28900: Make the objectstore-secondary connection pool size
configurable (#5764)
---
.../org/apache/hadoop/hive/metastore/conf/MetastoreConf.java | 5 +++++
.../hadoop/hive/metastore/PersistenceManagerProvider.java | 8 +++++---
.../metastore/datasource/TestDataSourceProviderFactory.java | 12 ++++++++++++
3 files changed, 22 insertions(+), 3 deletions(-)
diff --git
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index 7a6514d27fd..21ee4ce26c3 100644
---
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -700,6 +700,11 @@ public enum ConfVars {
"configured with embedded metastore. To get optimal performance,
set config to meet the following condition\n"+
"(2 * pool_size * metastore_instances + 2 * pool_size *
HS2_instances_with_embedded_metastore) = \n" +
"(2 * physical_core_count + hard_disk_count)."),
+
CONNECTION_POOLING_MAX_SECONDARY_CONNECTIONS("datanucleus.connectionPool.secondary.maxPoolSize",
+ "datanucleus.connectionPool.secondary.maxPoolSize", 2,
+ "Specify the maximum number of connections in the secondary
connection pools (objectstore-secondary, objectstore-compactor-secondary).\n" +
+ "Same as datanucleus.connectionPool.maxPoolSize, but for
the objectstore-secondary and objectstore-compactor-secondary pools. \n" +
+ "Values smaller than 2 are ignored."),
CONNECT_URL_HOOK("metastore.ds.connection.url.hook",
"hive.metastore.ds.connection.url.hook", "",
"Name of the hook to use for retrieving the JDO connection URL. If
empty, the value in javax.jdo.option.ConnectionURL is used"),
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PersistenceManagerProvider.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PersistenceManagerProvider.java
index 9afc7168aaa..daaed16d947 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PersistenceManagerProvider.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PersistenceManagerProvider.java
@@ -314,10 +314,12 @@ private static PersistenceManagerFactory
initPMF(Configuration conf, boolean for
ds = (maxPoolSize > 0) ? dsp.create(conf, maxPoolSize) :
dsp.create(conf);
databaseProduct = DatabaseProduct.determineDatabaseProduct(ds, conf);
// The secondary connection factory is used for schema generation, and
for value generation operations.
- // We should use a different pool for the secondary connection factory
to avoid resource starvation.
- // Since DataNucleus uses locks for schema generation and value
generation, 2 connections should be sufficient.
+ // We use a different pool for the secondary connection factory to
avoid resource starvation.
+ // DataNucleus uses locks for schema generation and value generation,
under normal circumstances 2 connections
+ // should be sufficient.
configurator.resetName(sourceName + "-secondary");
- ds2 = dsp.create(conf, /* maxPoolSize */ 2);
+ int maxSecondaryPoolSize = Math.max(2, MetastoreConf.getIntVar(conf,
ConfVars.CONNECTION_POOLING_MAX_SECONDARY_CONNECTIONS));
+ ds2 = dsp.create(conf, maxSecondaryPoolSize);
dsProp.put(PropertyNames.PROPERTY_CONNECTION_FACTORY, ds);
dsProp.put(PropertyNames.PROPERTY_CONNECTION_FACTORY2, ds2);
dsProp.put(ConfVars.MANAGER_FACTORY_CLASS.getVarname(),
diff --git
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java
index 90bd3aeb5d0..9e6690315cf 100644
---
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java
+++
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java
@@ -99,6 +99,18 @@ public void testSetHikariCpProperties() throws SQLException {
Assert.assertEquals(-1L, hds.getInitializationFailTimeout());
}
+ @Test
+ public void testHikariCpMaxPoolSize() {
+ MetastoreConf.setVar(conf, ConfVars.CONNECTION_POOLING_TYPE,
HikariCPDataSourceProvider.HIKARI);
+ PersistenceManagerProvider.updatePmfProperties(conf);
+ PersistenceManagerFactory pmf =
+
PersistenceManagerProvider.getPersistenceManager().getPersistenceManagerFactory();
+ HikariDataSource hikariDs = (HikariDataSource) pmf.getConnectionFactory();
+ HikariDataSource hikariDs2 = (HikariDataSource)
pmf.getConnectionFactory2();
+ Assert.assertEquals(hikariDs.getMaximumPoolSize(),
MetastoreConf.getIntVar(conf, ConfVars.CONNECTION_POOLING_MAX_CONNECTIONS));
+ Assert.assertEquals(hikariDs2.getMaximumPoolSize(),
MetastoreConf.getIntVar(conf,
ConfVars.CONNECTION_POOLING_MAX_SECONDARY_CONNECTIONS));
+ }
+
@Test
public void testCreateHikariCpDataSource() throws SQLException {
