Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id B53DF200C5E for ; Sat, 8 Apr 2017 06:24:55 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id B3C1D160BA2; Sat, 8 Apr 2017 04:24:55 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 07370160B97 for ; Sat, 8 Apr 2017 06:24:54 +0200 (CEST) Received: (qmail 33095 invoked by uid 500); 8 Apr 2017 04:24:54 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 33084 invoked by uid 99); 8 Apr 2017 04:24:53 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 08 Apr 2017 04:24:53 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 8F5B0E024D; Sat, 8 Apr 2017 04:24:53 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: prasanthj@apache.org To: commits@hive.apache.org Message-Id: <448a065803224ff0972088d74494ea9e@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hive git commit: HIVE-16383: Switch to HikariCP as default connection pooling (Prasanth Jayachandran reviewed by Thejas Nair) Date: Sat, 8 Apr 2017 04:24:53 +0000 (UTC) archived-at: Sat, 08 Apr 2017 04:24:55 -0000 Repository: hive Updated Branches: refs/heads/master 78d8b3377 -> 7629dbd5b HIVE-16383: Switch to HikariCP as default connection pooling (Prasanth Jayachandran reviewed by Thejas Nair) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7629dbd5 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7629dbd5 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7629dbd5 Branch: refs/heads/master Commit: 7629dbd5b722bc190d741f73c70bba2bd7daa87d Parents: 78d8b33 Author: Prasanth Jayachandran Authored: Fri Apr 7 21:24:43 2017 -0700 Committer: Prasanth Jayachandran Committed: Fri Apr 7 21:24:43 2017 -0700 ---------------------------------------------------------------------- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java | 9 ++++++++- .../org/apache/hadoop/hive/metastore/txn/TxnHandler.java | 4 +++- pom.xml | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/7629dbd5/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index dedf6a2..726522b 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -734,9 +734,16 @@ public class HiveConf extends Configuration { "Defaults to all permissions for the hiveserver2/metastore process user."), METASTORE_CACHE_PINOBJTYPES("hive.metastore.cache.pinobjtypes", "Table,StorageDescriptor,SerDeInfo,Partition,Database,Type,FieldSchema,Order", "List of comma separated metastore object types that should be pinned in the cache"), - METASTORE_CONNECTION_POOLING_TYPE("datanucleus.connectionPoolingType", "BONECP", new StringSet("BONECP", "DBCP", + METASTORE_CONNECTION_POOLING_TYPE("datanucleus.connectionPoolingType", "HikariCP", new StringSet("BONECP", "DBCP", "HikariCP", "NONE"), "Specify connection pool library for datanucleus"), + METASTORE_CONNECTION_POOLING_MAX_CONNECTIONS("datanucleus.connectionPool.maxPoolSize", 10, + "Specify the maximum number of connections in the connection pool. Note: The configured size will be used by" + + " 2 connection pools (TxnHandler and ObjectStore). When configuring the max connection pool size, it is " + + "recommended to take into account the number of metastore instances and the number of HiveServer2 instances " + + "configured with embedded metastore. To get optimal performance, set config to meet the following condition"+ + "(2 * pool_size * metastore_instances + 2 * pool_size * HS2_instances_with_embedded_metastore) = " + + "(2 * physical_core_count + hard_disk_count)."), // Workaround for DN bug on Postgres: // http://www.datanucleus.org/servlet/forum/viewthread_thread,7985_offset METASTORE_DATANUCLEUS_INIT_COL_INFO("datanucleus.rdbms.initializeColumnInfo", "NONE", http://git-wip-us.apache.org/repos/asf/hive/blob/7629dbd5/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java index 59461e6..14c834c 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java @@ -3139,6 +3139,7 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI { String passwd = getMetastoreJdbcPasswd(conf); String connectionPooler = conf.getVar( HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE).toLowerCase(); + int maxPoolSize = conf.getIntVar(HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_MAX_CONNECTIONS); if ("bonecp".equals(connectionPooler)) { BoneCPConfig config = new BoneCPConfig(); @@ -3146,8 +3147,8 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI { //if we are waiting for connection for 60s, something is really wrong //better raise an error than hang forever config.setConnectionTimeoutInMs(60000); - config.setMaxConnectionsPerPartition(10); config.setPartitionCount(1); + config.setMaxConnectionsPerPartition(maxPoolSize); config.setUser(user); config.setPassword(passwd); connPool = new BoneCPDataSource(config); @@ -3162,6 +3163,7 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI { connPool = new PoolingDataSource(objectPool); } else if ("hikaricp".equals(connectionPooler)) { HikariConfig config = new HikariConfig(); + config.setMaximumPoolSize(maxPoolSize); config.setJdbcUrl(driverUrl); config.setUsername(user); config.setPassword(passwd); http://git-wip-us.apache.org/repos/asf/hive/blob/7629dbd5/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 48345b9..a73fac0 100644 --- a/pom.xml +++ b/pom.xml @@ -147,7 +147,7 @@ 1.1.1 3.3.0 - 2.5.1 + 2.6.1 4.5.2 4.4.4