Repository: hive Updated Branches: refs/heads/master 78d8b3377 -> 7629dbd5b
HIVE-16383: Switch to HikariCP as default connection pooling (Prasanth Jayachandran reviewed by Thejas Nair) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7629dbd5 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7629dbd5 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7629dbd5 Branch: refs/heads/master Commit: 7629dbd5b722bc190d741f73c70bba2bd7daa87d Parents: 78d8b33 Author: Prasanth Jayachandran <prasan...@apache.org> Authored: Fri Apr 7 21:24:43 2017 -0700 Committer: Prasanth Jayachandran <prasan...@apache.org> Committed: Fri Apr 7 21:24:43 2017 -0700 ---------------------------------------------------------------------- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java | 9 ++++++++- .../org/apache/hadoop/hive/metastore/txn/TxnHandler.java | 4 +++- pom.xml | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/7629dbd5/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index dedf6a2..726522b 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -734,9 +734,16 @@ public class HiveConf extends Configuration { "Defaults to all permissions for the hiveserver2/metastore process user."), METASTORE_CACHE_PINOBJTYPES("hive.metastore.cache.pinobjtypes", "Table,StorageDescriptor,SerDeInfo,Partition,Database,Type,FieldSchema,Order", "List of comma separated metastore object types that should be pinned in the cache"), - METASTORE_CONNECTION_POOLING_TYPE("datanucleus.connectionPoolingType", "BONECP", new StringSet("BONECP", "DBCP", + METASTORE_CONNECTION_POOLING_TYPE("datanucleus.connectionPoolingType", "HikariCP", new StringSet("BONECP", "DBCP", "HikariCP", "NONE"), "Specify connection pool library for datanucleus"), + METASTORE_CONNECTION_POOLING_MAX_CONNECTIONS("datanucleus.connectionPool.maxPoolSize", 10, + "Specify the maximum number of connections in the connection pool. Note: The configured size will be used by" + + " 2 connection pools (TxnHandler and ObjectStore). When configuring the max connection pool size, it is " + + "recommended to take into account the number of metastore instances and the number of HiveServer2 instances " + + "configured with embedded metastore. To get optimal performance, set config to meet the following condition"+ + "(2 * pool_size * metastore_instances + 2 * pool_size * HS2_instances_with_embedded_metastore) = " + + "(2 * physical_core_count + hard_disk_count)."), // Workaround for DN bug on Postgres: // http://www.datanucleus.org/servlet/forum/viewthread_thread,7985_offset METASTORE_DATANUCLEUS_INIT_COL_INFO("datanucleus.rdbms.initializeColumnInfo", "NONE", http://git-wip-us.apache.org/repos/asf/hive/blob/7629dbd5/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java index 59461e6..14c834c 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java @@ -3139,6 +3139,7 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI { String passwd = getMetastoreJdbcPasswd(conf); String connectionPooler = conf.getVar( HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE).toLowerCase(); + int maxPoolSize = conf.getIntVar(HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_MAX_CONNECTIONS); if ("bonecp".equals(connectionPooler)) { BoneCPConfig config = new BoneCPConfig(); @@ -3146,8 +3147,8 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI { //if we are waiting for connection for 60s, something is really wrong //better raise an error than hang forever config.setConnectionTimeoutInMs(60000); - config.setMaxConnectionsPerPartition(10); config.setPartitionCount(1); + config.setMaxConnectionsPerPartition(maxPoolSize); config.setUser(user); config.setPassword(passwd); connPool = new BoneCPDataSource(config); @@ -3162,6 +3163,7 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI { connPool = new PoolingDataSource(objectPool); } else if ("hikaricp".equals(connectionPooler)) { HikariConfig config = new HikariConfig(); + config.setMaximumPoolSize(maxPoolSize); config.setJdbcUrl(driverUrl); config.setUsername(user); config.setPassword(passwd); http://git-wip-us.apache.org/repos/asf/hive/blob/7629dbd5/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 48345b9..a73fac0 100644 --- a/pom.xml +++ b/pom.xml @@ -147,7 +147,7 @@ <hbase.version>1.1.1</hbase.version> <!-- required for logging test to avoid including hbase which pulls disruptor transitively --> <disruptor.version>3.3.0</disruptor.version> - <hikaricp.version>2.5.1</hikaricp.version> + <hikaricp.version>2.6.1</hikaricp.version> <!-- httpcomponents are not always in version sync --> <httpcomponents.client.version>4.5.2</httpcomponents.client.version> <httpcomponents.core.version>4.4.4</httpcomponents.core.version>
