hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From h...@apache.org
Subject incubator-hawq git commit: HAWQ-412. Allocate query resource for multiple EXECUTIONs in explicit and implicit prepared statement
Date Thu, 18 Feb 2016 02:49:04 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/master 7545aadeb -> 5620300d1


HAWQ-412. Allocate query resource for multiple EXECUTIONs in explicit and implicit prepared
statement


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/5620300d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/5620300d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/5620300d

Branch: refs/heads/master
Commit: 5620300d161d5390fb1577f9852175d894d87d37
Parents: 7545aad
Author: Ruilong Huo <rhuo@pivotal.io>
Authored: Wed Feb 17 18:42:37 2016 -0800
Committer: Ruilong Huo <rhuo@pivotal.io>
Committed: Wed Feb 17 18:42:37 2016 -0800

----------------------------------------------------------------------
 src/backend/cdb/cdbdatalocality.c    | 91 +++++++++++++++++++++++++++++--
 src/backend/executor/spi.c           | 31 +++++++++--
 src/backend/nodes/copyfuncs.c        |  8 ++-
 src/backend/optimizer/plan/planner.c |  9 +--
 src/include/cdb/cdbdatalocality.h    | 17 ++++++
 src/include/executor/execdesc.h      | 14 +++++
 src/include/nodes/plannodes.h        |  1 +
 7 files changed, 157 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/5620300d/src/backend/cdb/cdbdatalocality.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbdatalocality.c b/src/backend/cdb/cdbdatalocality.c
index c3385b1..6fc04ae 100644
--- a/src/backend/cdb/cdbdatalocality.c
+++ b/src/backend/cdb/cdbdatalocality.c
@@ -454,6 +454,33 @@ static Relation_File** change_file_order_based_on_continuity(
 
 static int64 set_maximum_segment_volumn_parameter(Relation_Data *rel_data,
 		int host_num, double* maxSizePerSegment);
+
+/*
+ * saveQueryResourceParameters: save QueryResourceParameters
+ * in prepare statement along with query plan so that the query
+ * resource can be re-allocated during multiple executions of
+ * the plan
+ */
+void saveQueryResourceParameters(
+		QueryResourceParameters	*resource_parameters,
+		QueryResourceLife       life,
+		int32                   slice_size,
+		int64_t                 iobytes,
+		int                     max_target_segment_num,
+		int                     min_target_segment_num,
+		HostnameVolumnInfo      *vol_info,
+		int                     vol_info_size)
+
+{
+	resource_parameters->life = life;
+	resource_parameters->slice_size = slice_size;
+	resource_parameters->iobytes = iobytes;
+	resource_parameters->max_target_segment_num = max_target_segment_num;
+	resource_parameters->min_target_segment_num = min_target_segment_num;
+	resource_parameters->vol_info = vol_info;
+	resource_parameters->vol_info_size = vol_info_size;
+}
+
 /*
  * Setup /cleanup the memory context for this run
  * of data locality algorithm.
@@ -1892,7 +1919,7 @@ search_map_node(List *result, Oid rel_oid, int host_num,
 
 static List *
 post_process_assign_result(Split_Assignment_Result *assign_result) {
-	List *final_result = NULL;
+	List *final_result = NIL;
 	int i;
 
 	for (i = 0; i < assign_result->host_num; i++) {
@@ -3934,16 +3961,22 @@ static void cleanup_allocation_algorithm(
 SplitAllocResult *
 calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
 		List *fullRangeTable, GpPolicy *intoPolicy, int sliceNum) {
-	SplitAllocResult *result;
+	SplitAllocResult *result = NULL;
 	QueryResource *resource = NULL;
-	List *virtual_segments;
-	List *alloc_result;
+	QueryResourceParameters *resource_parameters = NULL;
+
+	List *virtual_segments = NIL;
+	List *alloc_result = NIL;
 	split_to_segment_mapping_context context;
 
 	int planner_segments = -1; /*virtual segments number for explain statement */
 
 	result = (SplitAllocResult *) palloc(sizeof(SplitAllocResult));
+	result->resource = NULL;
+	result->resource_parameters = NULL;
+	result->alloc_results = NIL;
 	result->relsType = NIL;
+	result->planner_segments = -1;
 	result->datalocalityInfo = makeStringInfo();
 
 	/* fake data locality */
@@ -3955,8 +3988,9 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
 		}
 	}
 
-	if ((Gp_role != GP_ROLE_DISPATCH)) {
+	if (Gp_role != GP_ROLE_DISPATCH) {
 		result->resource = NULL;
+		result->resource_parameters = NULL;
 		result->alloc_results = NIL;
 		result->relsType = NIL;
 		result->planner_segments = -1;
@@ -3967,6 +4001,26 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
 
 	init_datalocality_context(&context);
 
+	/*
+	 * Initialize QueryResourceParameters in QD
+	 *
+	 * We use CacheMemoryContext/TopMemoryContext here so that the
+	 * QueryResourceParameter can be available in the session. Thus,
+	 * it can be used in multiple "EXECUTION"s of the prepared
+	 * statement (i.e., "PREPARE", "BIND", "EXECUTION").
+	 */
+	MemoryContext oldcontext = NULL;
+	if ( CacheMemoryContext != NULL )
+	{
+		oldcontext = MemoryContextSwitchTo(CacheMemoryContext);
+	}
+	else
+	{
+		oldcontext = MemoryContextSwitchTo(TopMemoryContext);
+	}
+	resource_parameters = (QueryResourceParameters *)palloc(sizeof(QueryResourceParameters));
+	MemoryContextSwitchTo(oldcontext);
+
 	collect_range_tables(query, fullRangeTable, &(context.rtc_context));
 
 	bool isTableFunctionExists = false;
@@ -4007,6 +4061,18 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
 	/*use inherit resource*/
 	if (resourceLife == QRL_INHERIT) {
 		resource = AllocateResource(resourceLife, sliceNum, 0, 0, 0, NULL, 0);
+
+		saveQueryResourceParameters(
+		        resource_parameters,  /* resource_parameters */
+		        resourceLife,         /* life */
+		        sliceNum,             /* slice_size */
+		        0,                    /* iobytes */
+		        0,                    /* max_target_segment_num */
+		        0,                    /* min_target_segment_num */
+		        NULL,                 /* vol_info */
+		        0                     /* vol_info_size */ 
+		        );
+
 		if (resource != NULL) {
 			if ((context.keep_hash)
 					&& (list_length(resource->segments) != context.hashSegNum)) {
@@ -4155,6 +4221,18 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
 			resource = AllocateResource(QRL_ONCE, sliceNum, queryCost,
 					maxTargetSegmentNumber, minTargetSegmentNumber,
 					context.host_context.hostnameVolInfos, context.host_context.size);
+
+			saveQueryResourceParameters(
+			        resource_parameters,                   /* resource_parameters */
+			        QRL_ONCE,                              /* life */
+			        sliceNum,                              /* slice_size */
+			        queryCost,                             /* iobytes */
+			        maxTargetSegmentNumber,                /* max_target_segment_num */
+			        minTargetSegmentNumber,                /* min_target_segment_num */
+			        context.host_context.hostnameVolInfos, /* vol_info */
+			        context.host_context.size              /* vol_info_size */
+			        );
+			
 		}
 		/* for explain statement, we doesn't allocate resource physically*/
 		else {
@@ -4172,7 +4250,9 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
 
 		if (resource == NULL) {
 			result->resource = NULL;
+			result->resource_parameters = NULL;
 			result->alloc_results = NIL;
+			result->relsType = NIL;
 			result->planner_segments = planner_segments;
 			return result;
 		}
@@ -4209,6 +4289,7 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
 	alloc_result = run_allocation_algorithm(result, virtual_segments, &resource, &context);
 
 	result->resource = resource;
+	result->resource_parameters = resource_parameters;
 	result->alloc_results = alloc_result;
 	result->planner_segments = list_length(resource->segments);
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/5620300d/src/backend/executor/spi.c
----------------------------------------------------------------------
diff --git a/src/backend/executor/spi.c b/src/backend/executor/spi.c
index e07a479..07812aa 100644
--- a/src/backend/executor/spi.c
+++ b/src/backend/executor/spi.c
@@ -1795,19 +1795,42 @@ _SPI_execute_plan(_SPI_plan * plan, Datum *Values, const char *Nulls,
 			foreach(query_list_item, query_list)
 			{
 				Query	   *queryTree = (Query *) lfirst(query_list_item);
-				PlannedStmt *stmt;
+				PlannedStmt *originalStmt;
 				QueryDesc  *qdesc;
 				DestReceiver *dest;
 
-				stmt = (PlannedStmt*)lfirst(plan_list_item);
+				originalStmt = (PlannedStmt*)lfirst(plan_list_item);
 				plan_list_item = lnext(plan_list_item);
 
 				/*
 				 * Get copy of the queryTree and the plan since this may be modified further down.
 				 */
 				queryTree = copyObject(queryTree);
-				stmt = copyObject(stmt); 
-				
+				PlannedStmt *stmt = copyObject(originalStmt);
+
+				/*
+				 * We only allocate resource for multiple executions of queries, NOT for utility commands.
+				 * SELECT/INSERT are supported at present.
+				 */
+				if( (queryTree->commandType == CMD_SELECT) ||
+				    (queryTree->commandType == CMD_INSERT) )
+				{
+					if ( (Gp_role == GP_ROLE_DISPATCH) &&
+					     (stmt->resource == NULL) &&
+					     (stmt->resource_parameters != NULL) )
+					{
+						stmt->resource = AllocateResource(stmt->resource_parameters->life,
+						                        stmt->resource_parameters->slice_size,
+						                        stmt->resource_parameters->iobytes,
+						                        stmt->resource_parameters->max_target_segment_num,
+						                        stmt->resource_parameters->min_target_segment_num,
+						                        stmt->resource_parameters->vol_info,
+						                        stmt->resource_parameters->vol_info_size);
+					}
+
+					originalStmt->resource = NULL;
+				}
+
 				_SPI_current->processed = 0;
 				_SPI_current->lastoid = InvalidOid;
 				_SPI_current->tuptable = NULL;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/5620300d/src/backend/nodes/copyfuncs.c
----------------------------------------------------------------------
diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c
index 8f48008..6723b8a 100644
--- a/src/backend/nodes/copyfuncs.c
+++ b/src/backend/nodes/copyfuncs.c
@@ -216,7 +216,13 @@ _copyPlannedStmt(PlannedStmt *from)
 	COPY_SCALAR_FIELD(backoff_weight);
 	COPY_SCALAR_FIELD(query_mem);
 
-	COPY_SCALAR_FIELD(resource); // ?? What does this mean?
+	/*
+	 * A shallow copy of PlannedStmt that only copies the
+	 * reference of query resource and its parameters
+	 */
+	COPY_SCALAR_FIELD(resource);
+	COPY_SCALAR_FIELD(resource_parameters);
+
 	COPY_SCALAR_FIELD(planner_segments);
 
 	return newnode;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/5620300d/src/backend/optimizer/plan/planner.c
----------------------------------------------------------------------
diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c
index c6b2263..824558b 100644
--- a/src/backend/optimizer/plan/planner.c
+++ b/src/backend/optimizer/plan/planner.c
@@ -285,12 +285,12 @@ planner(Query *parse, int cursorOptions,
 	PlannedStmt *result = NULL;
 	instr_time	starttime, endtime;
 	ResourceNegotiatorResult *ppResult = (ResourceNegotiatorResult *) palloc(sizeof(ResourceNegotiatorResult));
-  SplitAllocResult initResult={NULL, NULL, -1};
-  ppResult->saResult = initResult;
-  ppResult->stmt =NULL;
+	SplitAllocResult initResult = {NULL, NULL, NIL, -1, NIL, NULL};
+	ppResult->saResult = initResult;
+	ppResult->stmt = NULL;
 	static int plannerLevel = 0;
 	bool resourceNegotiateDone = false;
-	QueryResource *savedQueryResource = GetActiveQueryResource();;
+	QueryResource *savedQueryResource = GetActiveQueryResource();
 	SetActiveRelType(NIL);
 
 	bool isDispatchParallel = false;
@@ -455,6 +455,7 @@ planner(Query *parse, int cursorOptions,
 		SetActiveQueryResource(savedQueryResource);
 
 		result->resource = ppResult->saResult.resource;
+		result->resource_parameters = ppResult->saResult.resource_parameters;
 		result->scantable_splits = ppResult->saResult.alloc_results;
 		result->planner_segments = ppResult->saResult.planner_segments;
 		result->datalocalityInfo = ppResult->saResult.datalocalityInfo;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/5620300d/src/include/cdb/cdbdatalocality.h
----------------------------------------------------------------------
diff --git a/src/include/cdb/cdbdatalocality.h b/src/include/cdb/cdbdatalocality.h
index 902fc18..82a9f2c 100644
--- a/src/include/cdb/cdbdatalocality.h
+++ b/src/include/cdb/cdbdatalocality.h
@@ -41,6 +41,7 @@
 typedef struct SplitAllocResult
 {
   QueryResource *resource;
+  QueryResourceParameters *resource_parameters;
   List *alloc_results;
   int planner_segments;
   List *relsType;// relation type after datalocality changing
@@ -65,6 +66,22 @@ typedef struct VirtualSegmentNode
 } VirtualSegmentNode;
 
 /*
+ * saveQueryResourceParameters: save QueryResourceParameters
+ * in prepare statement along with query plan so that the query
+ * resource can be re-allocated during multiple executions of
+ * the plan
+ */
+void saveQueryResourceParameters(
+		QueryResourceParameters	*resource_parameters,
+		QueryResourceLife       life,
+		int32                   slice_size,
+		int64_t                 iobytes,
+		int                     max_target_segment_num,
+		int                     min_target_segment_num,
+		HostnameVolumnInfo      *vol_info,
+		int                     vol_info_size);
+
+/*
  * calculate_planner_segment_num: based on the parse tree,
  * we calculate the appropriate planner segment_num.
  */

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/5620300d/src/include/executor/execdesc.h
----------------------------------------------------------------------
diff --git a/src/include/executor/execdesc.h b/src/include/executor/execdesc.h
index e832106..df6e33c 100644
--- a/src/include/executor/execdesc.h
+++ b/src/include/executor/execdesc.h
@@ -77,6 +77,20 @@ typedef struct HostnameVolumnInfo
 	int64 datavolumn;
 } HostnameVolumnInfo;
 
+/*
+ * structure for query resource parameters
+ */
+typedef struct QueryResourceParameters
+{
+	QueryResourceLife  life;
+	int32              slice_size;
+	int64_t            iobytes;
+	int                max_target_segment_num;
+	int                min_target_segment_num;
+	HostnameVolumnInfo *vol_info;
+	int                vol_info_size;
+} QueryResourceParameters;
+
 /* ----------------
  *		query descriptor:
  *

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/5620300d/src/include/nodes/plannodes.h
----------------------------------------------------------------------
diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h
index 0181a48..842795e 100644
--- a/src/include/nodes/plannodes.h
+++ b/src/include/nodes/plannodes.h
@@ -197,6 +197,7 @@ typedef struct PlannedStmt
 		QueryContextInfo * contextdisp; /* query context for dispatching */
 
 		struct QueryResource *resource;
+		struct QueryResourceParameters *resource_parameters;
 		int	planner_segments;
 
     /* The overall memory consumption account (i.e., outside of an operator) */


Mime
View raw message