hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j.@apache.org
Subject [12/17] incubator-hawq git commit: HAWQ-558. The query running on random table does not push resource manager to request resource from YARN as maimum possible resource consumption
Date Fri, 18 Mar 2016 23:04:13 GMT
HAWQ-558. The query running on random table does not push resource manager to request resource
from YARN as maimum possible resource consumption


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/b3a35218
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/b3a35218
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/b3a35218

Branch: refs/heads/fix-enable-orca
Commit: b3a352186cbe4ebadc5b76121cbefde474fc3a53
Parents: e82ac3f
Author: YI JIN <yjin@pivotal.io>
Authored: Fri Mar 18 14:13:34 2016 +1100
Committer: YI JIN <yjin@pivotal.io>
Committed: Fri Mar 18 14:13:34 2016 +1100

----------------------------------------------------------------------
 src/backend/resourcemanager/resourcemanager.c | 83 +++++++++++++---------
 1 file changed, 51 insertions(+), 32 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b3a35218/src/backend/resourcemanager/resourcemanager.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcemanager.c b/src/backend/resourcemanager/resourcemanager.c
index 5a01316..c585d40 100644
--- a/src/backend/resourcemanager/resourcemanager.c
+++ b/src/backend/resourcemanager/resourcemanager.c
@@ -78,6 +78,7 @@ void completeAllocRequestToBroker(int32_t 	 *reqmem,
 								  int32_t 	 *reqcore,
 								  List 		**preferred);
 void processResourceBrokerTasks(void);
+void generateResourceRequestToResourceBroker(void);
 void cleanupAllGRMContainers(void);
 bool cleanedAllGRMContainers(void);
 
@@ -610,12 +611,15 @@ int MainHandlerLoop(void)
 		/* STEP 5. Handle all submitted requests through socket clients. */
 		processSubmittedRequests();
 
-        /* STEP 6. Check timeout resource allocation and timeout queuing requests. */
+		/* STEP 6. Generate possible resource request to resource broker. */
+		generateResourceRequestToResourceBroker();
+
+        /* STEP 7. Check timeout resource allocation and timeout queuing requests. */
         timeoutDeadResourceAllocation();
         timeoutQueuedRequest();
 
 		/*
-		 * STEP 7. Check the status of all segment nodes, mark down if hasn't got
+		 * STEP 8. Check the status of all segment nodes, mark down if hasn't got
 		 * 		   IMAlive message for a pre-defined period.
 		 */
         uint64_t curtime = gettime_microsec();
@@ -628,12 +632,13 @@ int MainHandlerLoop(void)
 			PRESPOOL->LastCheckTime = curtime;
 		}
 
-		/* STEP 8. Move all accepted GRM containers into resource pool. */
+
+		/* STEP 9. Move all accepted GRM containers into resource pool. */
 		moveAllAcceptedGRMContainersToResPool();
 
 		/*
-		 * Check if should pause dispatching resource to queries to collect
-		 * resource back in order to return GRM containers.
+		 * STEP 10. Check if should pause dispatching resource to queries to
+		 * collect resource back in order to return GRM containers.
 		 */
 		if ( DRMGlobalInstance->ImpType != NONE_HAWQ2 &&
 			 PRESPOOL->AddPendingContainerCount == 0 &&
@@ -654,7 +659,8 @@ int MainHandlerLoop(void)
 			setForcedReturnGRMContainerCount();
 		}
 
-		/* STEP 9. Dispatch resource to queries and send the messages out.*/
+		/* STEP 11. Dispatch resource to queries and send the messages out.*/
+
         if ( PRESPOOL->Segments.NodeCount > 0 && PQUEMGR->RatioCount >
0 &&
 			 PQUEMGR->toRunQueryDispatch &&
 			 PQUEMGR->ForcedReturnGRMContainerCount == 0 &&
@@ -675,23 +681,23 @@ int MainHandlerLoop(void)
 						 PQUEMGR->ForcedReturnGRMContainerCount);
         }
 
-        /* STEP 10. Generate output content to client connections. */
+        /* STEP 12. Generate output content to client connections. */
         sendResponseToClients();
 
         /*
-         * STEP 11. Return containers to global resource manager if there some
+         * STEP 13. Return containers to global resource manager if there some
          * 			some idle resource.
          */
         timeoutIdleGRMResourceToRB();
 
-		/* STEP 12. Notify segments to increase resource quota. */
+		/* STEP 14. Notify segments to increase resource quota. */
 		notifyToBeAcceptedGRMContainersToRMSEG();
 
-        /* STEP 13. Notify segments to decrease resource. */
+        /* STEP 15. Notify segments to decrease resource. */
         notifyToBeKickedGRMContainersToRMSEG();
 
         /*
-         * STEP 14. Check slaves file if the content is not checked or is
+         * STEP 16. Check slaves file if the content is not checked or is
          * 			updated.
          */
         checkSlavesFile();
@@ -2881,26 +2887,7 @@ void processResourceBrokerTasks(void)
 			}
 		}
 
-		/* STEP 3. Generate GRM container allocation request. */
-		curtime = gettime_microsec();
-
-        if ( PRESPOOL->Segments.NodeCount > 0 && PQUEMGR->RatioCount >
0 )
-        {
-        	refreshMemoryCoreRatioLevelUsage(curtime);
-        	if ( curtime - PRESPOOL->LastResAcqTime > 1000000LL)
-        	{
-        		PRESPOOL->LastResAcqTime = curtime;
-				res = generateAllocRequestToBroker();
-				if ( res != FUNC_RETURN_OK )
-				{
-					elog(WARNING, "Resource manager fails to allocate container "
-								  "from global resource manager.");
-					goto exit;
-				}
-        	}
-        }
-
-        /* STEP 4. Return kicked GRM containers. */
+        /* STEP 3. Return kicked GRM containers. */
         curtime = gettime_microsec();
 
     	if ( !PRESPOOL->pausePhase[QUOTA_PHASE_KICKED_TO_RETURN] )
@@ -2918,7 +2905,7 @@ void processResourceBrokerTasks(void)
 			elog(LOG, "Paused returning GRM containers kicked to GRM.");
     	}
 	    /*
-	     * STEP 5. Handle resource broker input as new allocated resource or
+	     * STEP 4. Handle resource broker input as new allocated resource or
 		 * 		   cluster report, container report etc. The allocated resource
 		 * 		   from resource broker will be added to the resource queue
 		 * 		   manager and resource pool.
@@ -2940,6 +2927,38 @@ exit:
 	RB_handleError(res);
 }
 
+void generateResourceRequestToResourceBroker(void)
+{
+	uint64_t curtime = 0;
+	int		 res	 = FUNC_RETURN_OK;
+
+	if ( !isCleanGRMResourceStatus() )
+	{
+
+		curtime = gettime_microsec();
+
+        if ( PRESPOOL->Segments.NodeCount > 0 && PQUEMGR->RatioCount >
0 )
+        {
+        	refreshMemoryCoreRatioLevelUsage(curtime);
+        	if ( curtime - PRESPOOL->LastResAcqTime > 1000000LL)
+        	{
+        		PRESPOOL->LastResAcqTime = curtime;
+				res = generateAllocRequestToBroker();
+				if ( res != FUNC_RETURN_OK )
+				{
+					elog(WARNING, "Resource manager fails to allocate container "
+								  "from global resource manager.");
+					goto exit;
+				}
+        	}
+        }
+	}
+
+exit:
+	RB_handleError(res);
+
+}
+
 void cleanupAllGRMContainers(void)
 {
 	dropAllResPoolGRMContainersToToBeKicked();


Mime
View raw message