hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bhuvnesh2...@apache.org
Subject [04/12] incubator-hawq git commit: HAWQ-623. Resource quota request does not follow latest resource quota calculating logic
Date Wed, 06 Apr 2016 03:42:02 GMT
HAWQ-623. Resource quota request does not follow latest resource quota calculating logic


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/cf2744dc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/cf2744dc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/cf2744dc

Branch: refs/heads/HAWQ-617
Commit: cf2744dc5fdefb47a4fbcd6f8fa807861822ebbd
Parents: 3828d91
Author: YI JIN <yjin@pivotal.io>
Authored: Tue Apr 5 12:03:19 2016 +1000
Committer: YI JIN <yjin@pivotal.io>
Committed: Tue Apr 5 12:03:19 2016 +1000

----------------------------------------------------------------------
 src/backend/resourcemanager/requesthandler.c  | 73 +++++++++++++++++++---
 src/backend/resourcemanager/resqueuemanager.c | 40 +-----------
 2 files changed, 69 insertions(+), 44 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/cf2744dc/src/backend/resourcemanager/requesthandler.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/requesthandler.c b/src/backend/resourcemanager/requesthandler.c
index 4e24848..b89d23a 100644
--- a/src/backend/resourcemanager/requesthandler.c
+++ b/src/backend/resourcemanager/requesthandler.c
@@ -793,18 +793,52 @@ bool handleRMRequestAcquireResourceQuota(void **arg)
 	int      		 res		= FUNC_RETURN_OK;
 	ConnectionTrack  conntrack  = (ConnectionTrack)(*arg);
 	bool			 exist		= false;
+	uint64_t		 reqtime	= gettime_microsec();
+	/* If we run in YARN mode, we expect that we should try to get at least one
+	 * available segment, and this requires at least once global resource manager
+	 * cluster report returned.
+	 */
+	if ( reqtime - DRMGlobalInstance->ResourceManagerStartTime <=
+		 rm_nocluster_timeout * 1000000LL &&
+		 PRESPOOL->RBClusterReportCounter == 0 )
+	{
+		elog(DEBUG3, "Resource manager defers the resource request.");
+		return false;
+	}
+
+	/*
+	 * If resource queue has no concrete capacity set yet, no need to handle
+	 * the request.
+	 */
+	if ( PQUEMGR->RootTrack->QueueInfo->ClusterMemoryMB <= 0 )
+	{
+		elog(DEBUG3, "Resource manager defers the resource request because the "
+					 "resource queues have no valid resource capacities yet.");
+		return false;
+	}
+
+	Assert(PRESPOOL->SlavesHostCount > 0);
+	int rejectlimit = ceil(PRESPOOL->SlavesHostCount * rm_rejectrequest_nseg_limit);
+	int unavailcount = PRESPOOL->SlavesHostCount - PRESPOOL->AvailNodeCount;
+	if ( unavailcount > rejectlimit )
+	{
+		snprintf(errorbuf, sizeof(errorbuf),
+				 "%d of %d segments %s unavailable, exceeds %.1f%% defined in "
+				 "GUC hawq_rm_rejectrequest_nseg_limit. The resource quota "
+				 "request is rejected.",
+				 unavailcount,
+				 PRESPOOL->SlavesHostCount,
+				 unavailcount == 1 ? "is" : "are",
+				 rm_rejectrequest_nseg_limit*100.0);
+		elog(WARNING, "ConnID %d. %s", conntrack->ConnID, errorbuf);
+		res = RESOURCEPOOL_TOO_MANY_UAVAILABLE_HOST;
+		goto errorexit;
+	}
 
 	RPCRequestHeadAcquireResourceQuotaFromRMByOID request =
 		SMBUFF_HEAD(RPCRequestHeadAcquireResourceQuotaFromRMByOID,
 					&(conntrack->MessageBuff));
 
-	elog(LOG, "ConnID %d. User "INT64_FORMAT" acquires query resource quota "
-			  "with expected %d vseg (MIN %d).",
-			  conntrack->ConnID,
-			  request->UseridOid,
-			  request->MaxSegCountFix,
-			  request->MinSegCountFix);
-
 	/* Get user name from oid. */
 	UserInfo reguser = getUserByUserOID(request->UseridOid, &exist);
 	if ( !exist )
@@ -826,6 +860,31 @@ bool handleRMRequestAcquireResourceQuota(void **arg)
 	conntrack->MinSegCountFixed = request->MinSegCountFix;
 	conntrack->VSegLimitPerSeg	= request->VSegLimitPerSeg;
 	conntrack->VSegLimit		= request->VSegLimit;
+	conntrack->StatVSegMemoryMB	= request->StatVSegMemoryMB;
+	conntrack->StatNVSeg		= request->StatNVSeg;
+
+	elog(RMLOG, "ConnID %d. User "INT64_FORMAT" acquires query resource quota. "
+			  	"Expect %d vseg (MIN %d). "
+				"Each segment has maximum %d vseg. "
+				"Query has maximum %d vseg. "
+				"Statement quota %d MB x %d vseg",
+				conntrack->ConnID,
+				request->UseridOid,
+				request->MaxSegCountFix,
+				request->MinSegCountFix,
+				request->VSegLimitPerSeg,
+				request->VSegLimit,
+				request->StatVSegMemoryMB,
+				request->StatNVSeg);
+
+	if ( conntrack->StatNVSeg > 0 )
+	{
+		elog(LOG, "ConnID %d. Statement level resource quota is active. "
+				  "Expect resource ( %d MB ) x %d.",
+				  conntrack->ConnID,
+				  conntrack->StatVSegMemoryMB,
+				  conntrack->StatNVSeg);
+	}
 
 	res = acquireResourceQuotaFromResQueMgr(conntrack, errorbuf, sizeof(errorbuf));
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/cf2744dc/src/backend/resourcemanager/resqueuemanager.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resqueuemanager.c b/src/backend/resourcemanager/resqueuemanager.c
index eec2fb4..5d43f92 100644
--- a/src/backend/resourcemanager/resqueuemanager.c
+++ b/src/backend/resourcemanager/resqueuemanager.c
@@ -1958,21 +1958,10 @@ int acquireResourceQuotaFromResQueMgr(ConnectionTrack	conntrack,
 	userinfo = getUserByUserName(conntrack->UserID,
 							     strlen(conntrack->UserID),
 							     &exist);
-	if ( exist )
-	{
-		/* Get the queue, and check if the parallel limit is achieved. */
-		queuetrack = getQueueTrackByQueueOID(userinfo->QueueOID, &exist);
-	}
-	else
-	{
-		elog(LOG, "ConnID %d. No user %s defined for registering connection. "
-				  "Assign to pg_default queue.",
-				  conntrack->ConnID,
-				  conntrack->UserID);
-		queuetrack = PQUEMGR->DefaultTrack;
-		userinfo = NULL;
-	}
+	Assert(exist && userinfo != NULL);
 
+	/* Get the queue, and check if the parallel limit is achieved. */
+	queuetrack = getQueueTrackByQueueOID(userinfo->QueueOID, &exist);
 	Assert( queuetrack != NULL );
 
 	conntrack->QueueTrack = queuetrack;
@@ -1986,29 +1975,6 @@ int acquireResourceQuotaFromResQueMgr(ConnectionTrack	conntrack,
 		if ( conntrack->StatNVSeg == 0 )
 		{
 			/*------------------------------------------------------------------
-			 * The following logic consider the actual resource requirement from
-			 * dispatcher based on table size, workload, etc. The requirement is
-			 * described by (MinSegCountFixed, MaxSegCountFixed). The requirement
-			 * can be satisfied only when there is a non-empty intersect between
-			 * (MinSegCountFixed, MaxSegCountFixed) and (SegNumMin, SegNum).
-			 *------------------------------------------------------------------
-			 */
-			conntrack->SegNumMin = conntrack->MaxSegCountFixed < conntrack->SegNumMin
?
-								   conntrack->MinSegCountFixed :
-								   max(conntrack->SegNumMin, conntrack->MinSegCountFixed);
-
-			conntrack->SegNum = min(conntrack->SegNum, conntrack->MaxSegCountFixed);
-
-			Assert( conntrack->SegNumMin <= conntrack->SegNum );
-			elog(LOG, "ConnID %d. Query resource quota expects (%d MB, %lf CORE) x %d "
-					  "( MIN %d ) resource after adjusting based on query characters.",
-					  conntrack->ConnID,
-					  conntrack->SegMemoryMB,
-					  conntrack->SegCore,
-					  conntrack->SegNum,
-					  conntrack->SegNumMin);
-
-			/*------------------------------------------------------------------
 			 * Adjust the number of virtual segments again based on
 			 * NVSEG_*_LIMITs and NVSEG_*_LIMIT_PERSEGs. This adjustment must
 			 * succeed.


Mime
View raw message