hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lil...@apache.org
Subject [1/2] incubator-hawq git commit: HAWQ-592. QD shouldn't try to reconnect to segment when meeting invalid QE for queries which only need to dispatch to all cached QEs
Date Thu, 31 Mar 2016 04:18:49 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/master 6107a7ce3 -> ae8aff2ea


HAWQ-592. QD shouldn't try to reconnect to segment when meeting invalid QE for queries which
only need to dispatch to all cached QEs


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/84d40fa6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/84d40fa6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/84d40fa6

Branch: refs/heads/master
Commit: 84d40fa606b8e3d90faab68988bde64c2ddb8d1a
Parents: 6107a7c
Author: Lili Ma <ictmalili@gmail.com>
Authored: Wed Mar 30 15:33:37 2016 +0800
Committer: Lili Ma <ictmalili@gmail.com>
Committed: Thu Mar 31 12:10:53 2016 +0800

----------------------------------------------------------------------
 src/backend/cdb/dispatcher.c     | 41 +++++++++++++++++++++--------------
 src/backend/cdb/dispatcher_mgt.c | 14 +++++++++---
 src/backend/cdb/executormgr.c    |  5 +++++
 src/include/cdb/dispatcher_mgt.h |  3 ++-
 src/include/cdb/executormgr.h    |  1 +
 5 files changed, 44 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/84d40fa6/src/backend/cdb/dispatcher.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/dispatcher.c b/src/backend/cdb/dispatcher.c
index d2ee4b7..67530fd 100644
--- a/src/backend/cdb/dispatcher.c
+++ b/src/backend/cdb/dispatcher.c
@@ -946,22 +946,25 @@ dispatcher_bind_executor(DispatchData *data)
 			DispatchTask	*task = &slice->tasks[j];
 			struct SegmentDatabaseDescriptor *desc;
 
-			executor = dispmgt_get_query_executor_iterator(&iterator);
+			executor = dispmgt_get_query_executor_iterator(&iterator, false);
 			desc = executormgr_allocate_executor(task->segment, task->id.is_writer, task->entrydb);
 
 			if (!desc)
 			{
-				struct ConcurrentConnectExecutorInfo *info;
-
-				info = dispmgt_build_preconnect_info(task->segment,
-													task->id.is_writer,
-													executor,
-													data,
-													slice,
-													task);
-				concurrent_connect_executors = lappend(concurrent_connect_executors, info);
-				data->num_of_new_connected_executors++;
-				continue;
+			  // may have problem for tasks assigned to entrydb for dispatch_statement_node
+			  // and dispatch_statement_string
+			  if (task->segment != NULL)
+			  {
+			    struct ConcurrentConnectExecutorInfo *info;
+
+			    info = dispmgt_build_preconnect_info(task->segment,
+                                               task->id.is_writer, executor,
+                                               data, slice, task);
+			    concurrent_connect_executors = lappend(concurrent_connect_executors,
+                                                 info);
+			    data->num_of_new_connected_executors++;
+			  }
+			  continue;
 			}
 
 #ifdef FAULT_INJECTOR
@@ -1021,7 +1024,9 @@ dispatcher_serialize_state(DispatchData *data)
 		{
 			DispatchTask	*task = &slice->tasks[j];
 
-			executor = dispmgt_get_query_executor_iterator(&iterator);
+			executor = dispmgt_get_query_executor_iterator(&iterator, true);
+			if (executor == NULL)
+			  continue;
  			executormgr_serialize_executor_state(data, executor, task, slice);
 
 			if (sliceTable)
@@ -1094,7 +1099,9 @@ dispatcher_unbind_executor(DispatchData *data)
 		{
 			DispatchTask	*task = &slice->tasks[j];
 
-			executor = dispmgt_get_query_executor_iterator(&iterator);
+			executor = dispmgt_get_query_executor_iterator(&iterator, true);
+			if (executor == NULL)
+			  continue;
 			executormgr_unbind_executor_task(data, executor, task, slice);
 		}
 	}
@@ -1200,7 +1207,9 @@ dispatcher_update_statistics(DispatchData *data)
 
 		for (j = 0; j < slice->tasks_num; j++)
 		{
-			executor = dispmgt_get_query_executor_iterator(&iterator);
+			executor = dispmgt_get_query_executor_iterator(&iterator, true);
+			if (executor == NULL)
+			  continue;
 			dispatcher_update_statistics_per_executor(data, executor);
 		}
 	}
@@ -1739,7 +1748,7 @@ dispatch_collect_executors_error(DispatchData *data)
 	char **errHostName = (char**) palloc0(errHostSize * sizeof(char *));
 
 	dispmgt_init_query_executor_iterator(data->query_executor_team, &iterator);
-	while ((executor = dispmgt_get_query_executor_iterator(&iterator)) != NULL)
+	while ((executor = dispmgt_get_query_executor_iterator(&iterator, true)) != NULL)
 	{
 		executormgr_merge_error_for_dispatcher(executor, &errHostSize,
 		                                       &errHostNum, &errHostName);

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/84d40fa6/src/backend/cdb/dispatcher_mgt.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/dispatcher_mgt.c b/src/backend/cdb/dispatcher_mgt.c
index d0fd7e7..7cf2573 100644
--- a/src/backend/cdb/dispatcher_mgt.c
+++ b/src/backend/cdb/dispatcher_mgt.c
@@ -144,6 +144,9 @@ dispmgt_get_query_executor_in_group_iterator(QueryExecutorGroup *group,
 			executormgr_is_stop(executor))
 			continue;
 
+		if (!executormgr_is_executor_valid(executor))
+		  continue;
+
 		/* Increase index or we may return same object in the next iterate. */
 		iterator->executor_id++;
 		return executor;
@@ -162,9 +165,10 @@ dispmgt_init_query_executor_iterator(QueryExecutorTeam *team,
 }
 
 struct QueryExecutor *
-dispmgt_get_query_executor_iterator(QueryExecutorIterator *iterator)
+dispmgt_get_query_executor_iterator(QueryExecutorIterator *iterator, bool mayContainInvalidExecutor)
 {
 	QueryExecutorGroup	*group;
+	struct QueryExecutor *executor = NULL;
 
 	do {
 		if (iterator->group_id >= iterator->team->query_executor_group_num)
@@ -180,7 +184,11 @@ dispmgt_get_query_executor_iterator(QueryExecutorIterator *iterator)
 			continue;
 		}
 
-		return group->query_executors[iterator->executor_id++];
+		executor = group->query_executors[iterator->executor_id++];
+		if (mayContainInvalidExecutor && !executormgr_is_executor_valid(executor))
+		  continue;
+		else
+		  return executor;
 	} while (1);
 
 	return NULL;
@@ -194,7 +202,7 @@ dispmgt_takeover_segment_conns(QueryExecutorTeam *team)
 	List					*segment_conns = NIL;
 
 	dispmgt_init_query_executor_iterator(team, &iterator);
-	while ((executor = dispmgt_get_query_executor_iterator(&iterator)))
+	while ((executor = dispmgt_get_query_executor_iterator(&iterator, true)))
 	{
 		segment_conns = lappend(segment_conns, executormgr_takeover_segment_conns(executor));
 	}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/84d40fa6/src/backend/cdb/executormgr.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/executormgr.c b/src/backend/cdb/executormgr.c
index 2bc5059..7150346 100644
--- a/src/backend/cdb/executormgr.c
+++ b/src/backend/cdb/executormgr.c
@@ -664,6 +664,11 @@ executormgr_is_executor_error(QueryExecutor *executor) {
     return false;
 }
 
+bool
+executormgr_is_executor_valid(QueryExecutor *executor) {
+  return executor->desc != NULL;
+}
+
 SegmentDatabaseDescriptor *
 executormgr_takeover_segment_conns(QueryExecutor *executor)
 {

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/84d40fa6/src/include/cdb/dispatcher_mgt.h
----------------------------------------------------------------------
diff --git a/src/include/cdb/dispatcher_mgt.h b/src/include/cdb/dispatcher_mgt.h
index f5ca7b7..e386a47 100644
--- a/src/include/cdb/dispatcher_mgt.h
+++ b/src/include/cdb/dispatcher_mgt.h
@@ -64,7 +64,8 @@ typedef struct QueryExecutorTeam {
 extern void dispmgt_init_query_executor_iterator(struct QueryExecutorTeam *team,
 							QueryExecutorIterator *iterator);
 extern struct QueryExecutor *dispmgt_get_query_executor_iterator(
-							QueryExecutorIterator *iterator);
+							QueryExecutorIterator *iterator,
+							bool mayContainInvalidExecutor);
 extern int	dispmgt_get_group_num(struct QueryExecutorTeam *team);
 
 extern struct QueryExecutorTeam *dispmgt_create_dispmgt_state(struct DispatchData *data,

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/84d40fa6/src/include/cdb/executormgr.h
----------------------------------------------------------------------
diff --git a/src/include/cdb/executormgr.h b/src/include/cdb/executormgr.h
index e7fe9a0..d1c562e 100644
--- a/src/include/cdb/executormgr.h
+++ b/src/include/cdb/executormgr.h
@@ -80,6 +80,7 @@ extern void executormgr_merge_error_for_dispatcher(
     struct QueryExecutor *executor, int *errHostSize,
     int *errNum, char ***errHostInfo);
 extern bool executormgr_is_executor_error(struct QueryExecutor *executor);
+extern bool executormgr_is_executor_valid(struct QueryExecutor *executor);
 extern void executormgr_setup_env(MemoryContext ctx);
 extern void executormgr_cleanup_env(void);
 extern int executormgr_get_cached_executor_num(void);


Mime
View raw message