hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From h...@apache.org
Subject [hawq] 01/01: HAWQ-1748. Idle executors keep exist instead of quit even when gp_vmem_idle_resource_timeout specified timer expires
Date Thu, 15 Aug 2019 10:30:59 GMT
This is an automated email from the ASF dual-hosted git repository.

huor pushed a commit to branch huor
in repository https://gitbox.apache.org/repos/asf/hawq.git

commit cfec1150eba4ba7d757cfea3fd7e884b765505b4
Author: Ruilong Huo <huoruilong@163.com>
AuthorDate: Thu Aug 15 18:21:11 2019 +0800

    HAWQ-1748. Idle executors keep exist instead of quit even when gp_vmem_idle_resource_timeout
specified timer expires
---
 depends/dbcommon/Makefile.global |  8 ++++----
 depends/storage/Makefile.global  |  8 ++++----
 depends/univplan/Makefile.global |  8 ++++----
 src/backend/cdb/executormgr.c    | 24 +++++++++++++++++++++++
 src/backend/cdb/poolmgr.c        | 42 ++++++++++++++++++++++++++++++++++++++++
 src/backend/storage/lmgr/proc.c  |  4 ++--
 src/backend/tcop/postgres.c      | 10 +++++++++-
 src/backend/utils/misc/guc.c     |  2 +-
 src/include/cdb/executormgr.h    |  3 +++
 src/include/cdb/poolmgr.h        |  2 ++
 10 files changed, 95 insertions(+), 16 deletions(-)

diff --git a/depends/dbcommon/Makefile.global b/depends/dbcommon/Makefile.global
index e471d53..9ebc256 100644
--- a/depends/dbcommon/Makefile.global
+++ b/depends/dbcommon/Makefile.global
@@ -20,15 +20,15 @@
 # A makefile that integrate building this module with hawq
 #------------------------------------------------------------------------------
 
-prefix := /usr/local/hawq
-enable_debug    = no
+prefix := /data/workspace/hawq
+enable_debug    = yes
 enable_coverage = no
 with_dbcommon = yes
 
 # Support for VPATH builds
 vpath_build = no
-abs_top_srcdir = /data/apache-hawq
-abs_top_builddir = /data/apache-hawq
+abs_top_srcdir = /data/repository/apache-hawq
+abs_top_builddir = /data/repository/apache-hawq
 
 ifneq ($(vpath_build),yes)
 top_srcdir = $(top_builddir)
diff --git a/depends/storage/Makefile.global b/depends/storage/Makefile.global
index 8fd25bf..4be6114 100644
--- a/depends/storage/Makefile.global
+++ b/depends/storage/Makefile.global
@@ -20,15 +20,15 @@
 # A makefile that integrate building this module with hawq
 #------------------------------------------------------------------------------
 
-prefix := /usr/local/hawq
-enable_debug    = no
+prefix := /data/workspace/hawq
+enable_debug    = yes
 enable_coverage = no
 with_storage = yes
 
 # Support for VPATH builds
 vpath_build = no
-abs_top_srcdir = /data/apache-hawq
-abs_top_builddir = /data/apache-hawq
+abs_top_srcdir = /data/repository/apache-hawq
+abs_top_builddir = /data/repository/apache-hawq
 
 ifneq ($(vpath_build),yes)
 top_srcdir = $(top_builddir)
diff --git a/depends/univplan/Makefile.global b/depends/univplan/Makefile.global
index a291648..31c56ce 100644
--- a/depends/univplan/Makefile.global
+++ b/depends/univplan/Makefile.global
@@ -20,15 +20,15 @@
 # A makefile that integrate building this module with hawq
 #------------------------------------------------------------------------------
 
-prefix := /usr/local/hawq
-enable_debug    = no
+prefix := /data/workspace/hawq
+enable_debug    = yes
 enable_coverage = no
 with_univplan = yes
 
 # Support for VPATH builds
 vpath_build = no
-abs_top_srcdir = /data/apache-hawq
-abs_top_builddir = /data/apache-hawq
+abs_top_srcdir = /data/repository/apache-hawq
+abs_top_builddir = /data/repository/apache-hawq
 
 ifneq ($(vpath_build),yes)
 top_srcdir = $(top_builddir)
diff --git a/src/backend/cdb/executormgr.c b/src/backend/cdb/executormgr.c
index 5a1c15d..62473d7 100644
--- a/src/backend/cdb/executormgr.c
+++ b/src/backend/cdb/executormgr.c
@@ -1096,3 +1096,27 @@ executormgr_destory(SegmentDatabaseDescriptor *desc)
 	pfree(desc);
 }
 
+bool executormgr_has_cached_executor()
+{
+    return executor_cache.cached_num > 0;
+}
+
+bool executormgr_clean_cached_executor_filter(PoolItem item)
+{
+    SegmentDatabaseDescriptor *desc = (SegmentDatabaseDescriptor *)item;
+    return desc->conn->asyncStatus == PGASYNC_IDLE;
+}
+
+void executormgr_clean_cached_executor()
+{
+    /* go through each cached executor */
+    int cleaned = 0;
+    if (!executor_cache.init)
+    {
+        return;
+    }
+
+    cleaned = poolmgr_clean(executor_cache.pool, (PoolMgrIterateFilter) executormgr_clean_cached_executor_filter);
+    elog(DEBUG5, "cleaned %d idle executors", cleaned);
+}
+
diff --git a/src/backend/cdb/poolmgr.c b/src/backend/cdb/poolmgr.c
index eae0f04..f7194f6 100644
--- a/src/backend/cdb/poolmgr.c
+++ b/src/backend/cdb/poolmgr.c
@@ -184,3 +184,45 @@ poolmgr_iterate(PoolMgrState *pool, PoolMgrIterateFilter filter, PoolMgrIterateC
 	return;
 }
 
+int poolmgr_clean(struct PoolMgrState *pool, PoolMgrIterateFilter filter)
+{
+    int res = 0;
+    HASH_SEQ_STATUS hash_seq;
+    PoolItemEntry *entry;
+
+    if (!pool)
+        return res;
+
+    hash_seq_init(&hash_seq, pool->hashtable);
+    while ((entry = hash_seq_search(&hash_seq)))
+    {
+        ListCell *curr = list_head(entry->list);
+        ListCell *prev = NULL;
+        while (curr != NULL)
+        {
+            PoolItem  item = lfirst(curr);
+            if (filter && !filter(item))
+            {
+                /* try next */
+                prev = curr;
+                curr = lnext(prev);
+                continue;
+            }
+
+            /* clean now */
+            res++;
+            pool->callback(item);
+            entry->list = list_delete_cell(entry->list, curr, prev);
+            if (prev != NULL)
+            {
+                curr = lnext(prev);
+            }
+            else
+            {
+                curr = list_head(entry->list);
+            }
+        }
+    }
+    return res;
+}
+
diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c
index 2d3097f..4fef10c 100644
--- a/src/backend/storage/lmgr/proc.c
+++ b/src/backend/storage/lmgr/proc.c
@@ -1399,9 +1399,9 @@ HandleClientWaitTimeout(void)
 	/*
 	 * Free gangs to free up resources on the segDBs.
 	 */
-	if (gangsExist())
+	if (executormgr_has_cached_executor())
 	{
-		cleanupAllIdleGangs();
+		executormgr_clean_cached_executor();
 	}
 
 }
diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c
index cf0123e..c122280 100644
--- a/src/backend/tcop/postgres.c
+++ b/src/backend/tcop/postgres.c
@@ -4804,9 +4804,17 @@ PostgresMain(int argc, char *argv[], const char *username)
 			 * This means giving the end user enough time to type in the next SQL statement
 			 *
 			 */
-			if (IdleSessionGangTimeout > 0 && gangsExist())
+			if (IdleSessionGangTimeout > 0 && executormgr_has_cached_executor())
+            {
 				if (!enable_sig_alarm( IdleSessionGangTimeout /* ms */, false))
+                {
 					elog(FATAL, "could not set timer for client wait timeout");
+                }
+            }
+            else if (IdleSessionGangTimeout == 0)
+            {
+                executormgr_clean_cached_executor();
+            }
 		}
 
 		IdleTracker_DeactivateProcess();
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 0cf51df..b90f1ec 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -5067,7 +5067,7 @@ static struct config_int ConfigureNamesInt[] =
 	{
 		{"gp_vmem_idle_resource_timeout", PGC_USERSET, CLIENT_CONN_OTHER,
 			gettext_noop("Sets the time a session can be idle (in milliseconds) before we release
gangs on the segment DBs to free resources."),
-			gettext_noop("A value of 0 turns off the timeout."),
+			gettext_noop("A value of 0 closes idle gangs at once."),
 			GUC_UNIT_MS | GUC_GPDB_ADDOPT
 		},
 		&IdleSessionGangTimeout,
diff --git a/src/include/cdb/executormgr.h b/src/include/cdb/executormgr.h
index 31d44fa..ce4442d 100644
--- a/src/include/cdb/executormgr.h
+++ b/src/include/cdb/executormgr.h
@@ -98,5 +98,8 @@ extern bool executormgr_connect(struct SegmentDatabaseDescriptor *desc,
 							bool is_writer, bool is_superuser);
 extern void executormgr_free_executor(struct SegmentDatabaseDescriptor *desc);
 
+extern bool executormgr_has_cached_executor();
+extern void executormgr_clean_cached_executor();
+
 #endif	/* EXECUTORMGR_H */
 
diff --git a/src/include/cdb/poolmgr.h b/src/include/cdb/poolmgr.h
index 2821dd9..8408c25 100644
--- a/src/include/cdb/poolmgr.h
+++ b/src/include/cdb/poolmgr.h
@@ -32,6 +32,8 @@ typedef bool (*PoolMgrIterateFilter) (PoolItem item);
 
 extern struct PoolMgrState *poolmgr_create_pool(MemoryContext ctx, PoolMgrCleanCallback callback);
 extern bool poolmgr_drop_pool(struct PoolMgrState *pool);
+extern int poolmgr_clean(struct PoolMgrState *pool, PoolMgrIterateFilter filter);
+
 extern PoolItem poolmgr_get_item_by_name(struct PoolMgrState *pool, const char *name);
 extern PoolItem poolmgr_get_random_item(struct PoolMgrState *pool);
 extern void poolmgr_put_item(struct PoolMgrState *pool, const char *name, PoolItem item);


Mime
View raw message