hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From y...@apache.org
Subject incubator-hawq git commit: HAWQ-645. shrink LOG level log output from resource manager
Date Sat, 09 Apr 2016 02:36:29 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/master 143c0e3f1 -> 37839128b


HAWQ-645. shrink LOG level log output from resource manager


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/37839128
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/37839128
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/37839128

Branch: refs/heads/master
Commit: 37839128bfe8b063918300a929825485c1d0b3af
Parents: 143c0e3
Author: YI JIN <yjin@pivotal.io>
Authored: Sat Apr 9 12:36:12 2016 +1000
Committer: YI JIN <yjin@pivotal.io>
Committed: Sat Apr 9 12:36:12 2016 +1000

----------------------------------------------------------------------
 .../communication/rmcomm_Message.c              |  10 +-
 .../communication/rmcomm_QD2RM.c                |  20 +--
 src/backend/resourcemanager/requesthandler.c    |   4 +-
 .../resourcebroker/resourcebroker_LIBYARN.c     |  67 ++++++---
 .../resourcebroker_LIBYARN_proc.c               | 149 ++++++++++---------
 src/backend/resourcemanager/resourcemanager.c   |  22 +--
 src/backend/resourcemanager/resourcepool.c      |  66 ++++----
 src/backend/resourcemanager/resqueuemanager.c   |  62 ++++----
 src/backend/tcop/pquery.c                       |   2 -
 9 files changed, 224 insertions(+), 178 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/37839128/src/backend/resourcemanager/communication/rmcomm_Message.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/communication/rmcomm_Message.c b/src/backend/resourcemanager/communication/rmcomm_Message.c
index 364c8a2..b12a734 100644
--- a/src/backend/resourcemanager/communication/rmcomm_Message.c
+++ b/src/backend/resourcemanager/communication/rmcomm_Message.c
@@ -96,13 +96,9 @@ void ReadPostHandler_Message(AsyncCommBuffer buffer)
 			if ( DRM_MSGFRAME_TAILTAG_MATCHED(p) )
 			{
 				/* Skip heart-beat message log. */
-				elog((header->MessageID == REQUEST_QD_REFRESH_RESOURCE ||
-					  header->MessageID == REQUEST_RM_IMALIVE          ||
-					  header->MessageID == RESPONSE_QD_REFRESH_RESOURCE ||
-					  header->MessageID == RESPONSE_RM_IMALIVE) ? DEBUG3 : LOG,
-					 "AsyncComm framework receives message %d from FD %d",
-					 header->MessageID,
-					 buffer->FD);
+				elog(RMLOG, "AsyncComm framework receives message %d from FD %d",
+							header->MessageID,
+							buffer->FD);
 
 				/* Get complete message and call the handler. */
 				if ( context->MessageRecvedHandler != NULL )

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/37839128/src/backend/resourcemanager/communication/rmcomm_QD2RM.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/communication/rmcomm_QD2RM.c b/src/backend/resourcemanager/communication/rmcomm_QD2RM.c
index 4808a8b..f5cd379 100644
--- a/src/backend/resourcemanager/communication/rmcomm_QD2RM.c
+++ b/src/backend/resourcemanager/communication/rmcomm_QD2RM.c
@@ -698,16 +698,16 @@ int acquireResourceFromRM(int 		  		  index,
     								0 :
 									preferred_nodes_size;
 
-    elog(LOG, "ConnID: %d. Acquire resource request for index %d. "
-    		  "Max vseg size %d Min vseg size %d Estimated slice size %d "
-    		  "estimated IO bytes size " INT64_FORMAT " Preferred node count %d.",
-			  curcontext->QD_Conn_ID,
-			  index,
-			  max_seg_count_fix,
-			  min_seg_count_fix,
-			  slice_size,
-			  iobytes,
-			  nodecount);
+    elog(RMLOG, "ConnID: %d. Acquire resource request for index %d. "
+    		  	"Max vseg size %d Min vseg size %d Estimated slice size %d "
+    			"estimated IO bytes size " INT64_FORMAT " Preferred node count %d.",
+				curcontext->QD_Conn_ID,
+				index,
+				max_seg_count_fix,
+				min_seg_count_fix,
+				slice_size,
+				iobytes,
+				nodecount);
 
     /* Build request. */
     resetSelfMaintainBuffer(sendbuffer);

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/37839128/src/backend/resourcemanager/requesthandler.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/requesthandler.c b/src/backend/resourcemanager/requesthandler.c
index daef075..eb9fc1d 100644
--- a/src/backend/resourcemanager/requesthandler.c
+++ b/src/backend/resourcemanager/requesthandler.c
@@ -178,8 +178,8 @@ bool handleRMRequestConnectionRegByOID(void **arg)
 		if ( res == FUNC_RETURN_OK )
 		{
 			trackConnectionTrack(conntrack);
-			elog(LOG, "ConnID %d. Resource manager tracked connection.",
-					  conntrack->ConnID);
+			elog(RMLOG, "ConnID %d. Resource manager tracked connection.",
+					  	conntrack->ConnID);
 			response.Result = FUNC_RETURN_OK;
 			response.ConnID = conntrack->ConnID;
 		}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/37839128/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
index b8d205e..37e3bca 100644
--- a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
+++ b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
@@ -454,8 +454,8 @@ int RB_LIBYARN_getContainerReport(List **ctnstatl)
 		res = RESBROK_PIPE_ERROR;
 	}
 	destroySelfMaintainBuffer(&sendBuffer);
-	elog(LOG, "YARN mode resource broker wrote get container status request to "
-			  "resource broker process.");
+	elog(RMLOG, "YARN mode resource broker wrote get container status request to "
+			    "resource broker process.");
 	PipeReceivePending = res == FUNC_RETURN_OK;
 	return res;
 }
@@ -509,7 +509,7 @@ int RB_LIBYARN_handleNotification(void)
 		}
 
 		PipeReceivePending = false;
-		elog(LOG, "Finish processing message %d", messageid);
+		elog(RMLOG, "Finish processing message %d", messageid);
 
 		if ( res != FUNC_RETURN_OK )
 		{
@@ -568,9 +568,11 @@ int handleRB2RM_ClusterReport(void)
 	}
 
 	elog(LOG, "YARN mode resource broker got cluster report having %d host(s), "
-			  "maximum queue capacity is %lf.",
+			  "queue capacity %lf (MAX %lf) current capacity %lf.",
 			  response.MachineCount,
-			  response.QueueMaxCapacity);
+			  response.QueueCapacity,
+			  response.QueueMaxCapacity,
+			  response.QueueCurCapacity);
 
 	/* Read machines. */
 	for( int i = 0 ; i < response.MachineCount ; ++i )
@@ -659,12 +661,20 @@ int handleRB2RM_ClusterReport(void)
 
 	setAllSegResourceGRMUnhandled();
 
+
+	ResourceBundleData oldGRMTotalAll;
+	ResourceBundleData oldGRMTotal;
+	resetResourceBundleDataByBundle(&oldGRMTotalAll,
+									&(PRESPOOL->GRMTotalHavingNoHAWQNode));
+	resetResourceBundleDataByBundle(&oldGRMTotal,
+									&(PRESPOOL->GRMTotal));
 	/*
 	 * Start to update resource pool content. The YARN cluster total size is
 	 * also counted the same time.
 	 */
-
 	resetResourceBundleData(&(PRESPOOL->GRMTotalHavingNoHAWQNode), 0, 0.0, 0);
+	int updatedcnt = 0;
+	int skipedcnt = 0;
 
 	MEMORY_CONTEXT_SWITCH_TO(PCONTEXT)
 	while( list_length(segstats) > 0 )
@@ -681,20 +691,28 @@ int handleRB2RM_ClusterReport(void)
 			SelfMaintainBufferData buffer;
 			initializeSelfMaintainBuffer(&buffer, PCONTEXT);
 			generateSegStatReport(segstat, &buffer);
-			elog(LOG, "YARN mode resource broker updated segment configuration "
-					  "in resource pool. %s", buffer.Buffer);
+			elog(RMLOG, "YARN mode resource broker updated segment configuration "
+					    "in resource pool. %s", buffer.Buffer);
 			destroySelfMaintainBuffer(&buffer);
+			updatedcnt++;
 		}
 		else
 		{
-			elog(LOG, "YARN mode resource broker skipped segment configuration "
-					  "from host %s",
-					  GET_SEGINFO_GRMHOSTNAME(&(segstat->Info)));
+			elog(WARNING, "YARN mode resource broker skipped segment configuration "
+					  	  "from host %s",
+						  GET_SEGINFO_GRMHOSTNAME(&(segstat->Info)));
+			skipedcnt++;
 		}
 		rm_pfree(PCONTEXT, segstat);
 		segstats = list_delete_first(segstats);
 	}
 
+	elog(LOG, "YARN mode resource broker updated %d %s, skiped %d %s",
+			  updatedcnt,
+			  updatedcnt <= 1 ? "segment" : "segments",
+			  skipedcnt,
+			  skipedcnt <= 1 ? "segment" : "segments");
+
 	/*
 	 * iterate all segments without GRM report,
 	 * and update its status.
@@ -754,13 +772,26 @@ int handleRB2RM_ClusterReport(void)
 
 	MEMORY_CONTEXT_SWITCH_BACK
 
-	elog(LOG, "Resource manager YARN resource broker counted HAWQ cluster now "
-			  "having (%d MB, %lf CORE) in a YARN cluster of total resource "
-			  "(%d MB, %lf CORE).",
-			  PRESPOOL->GRMTotal.MemoryMB,
-			  PRESPOOL->GRMTotal.Core,
-			  PRESPOOL->GRMTotalHavingNoHAWQNode.MemoryMB,
-			  PRESPOOL->GRMTotalHavingNoHAWQNode.Core);
+	if ( oldGRMTotalAll.MemoryMB != PRESPOOL->GRMTotalHavingNoHAWQNode.MemoryMB ||
+		 oldGRMTotalAll.Core	 != PRESPOOL->GRMTotalHavingNoHAWQNode.Core )
+	{
+		elog(LOG, "Resource manager YARN resource broker counted YARN cluster "
+				  "having total resource (%d MB, %lf CORE).",
+				  PRESPOOL->GRMTotalHavingNoHAWQNode.MemoryMB,
+				  PRESPOOL->GRMTotalHavingNoHAWQNode.Core);
+	}
+
+	if ( oldGRMTotal.MemoryMB != PRESPOOL->GRMTotal.MemoryMB ||
+		 oldGRMTotal.Core != PRESPOOL->GRMTotal.Core )
+	{
+		elog(LOG, "Resource manager YARN resource broker counted HAWQ cluster now "
+				  "having (%d MB, %lf CORE) in a YARN cluster of total resource "
+				  "(%d MB, %lf CORE).",
+				  PRESPOOL->GRMTotal.MemoryMB,
+				  PRESPOOL->GRMTotal.Core,
+				  PRESPOOL->GRMTotalHavingNoHAWQNode.MemoryMB,
+				  PRESPOOL->GRMTotalHavingNoHAWQNode.Core);
+	}
 
 	/*
 	 * If the segment is GRM unavailable or FTS unavailable,

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/37839128/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN_proc.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN_proc.c b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN_proc.c
index ad49aae..5a6e6a2 100644
--- a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN_proc.c
+++ b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN_proc.c
@@ -189,8 +189,8 @@ int ResBrokerMainInternal(void)
 		 * run anymore. The process goes to the exit phase directly.
 		 */
 		if ( getppid() != ResBrokerParentPID ) {
-			elog(LOG, "Parent process of YARN mode resource broker quited. "
-					  "Resource broker process will actively close.");
+			elog(WARNING, "Parent process of YARN mode resource broker quit. "
+					  	  "Resource broker process will actively close.");
 			ResBrokerKeepRun = false;
 			continue;
 		}
@@ -270,10 +270,10 @@ int ResBrokerMainInternal(void)
 			}
 
             if ( res != FUNC_RETURN_OK ) {
-            	elog(LOG, "YARN mode resource broker failed to process request. "
-            			  "Message id = %d, result = %d.",
-            			  messageid,
-            			  res);
+            	elog(WARNING, "YARN mode resource broker failed to process request. "
+            			  	  "Message id = %d, result = %d.",
+							  messageid,
+							  res);
             	/* If this is a pipe error between RM and RB or YARN remove error.
             	 * Exit RB and let RM restart RB process. */
                 if ( res == RESBROK_ERROR_GRM )
@@ -1035,9 +1035,9 @@ int handleRM2RB_ReturnResource(void)
 
     for ( int i = 0 ; i < request.ContainerCount ; ++i )
     {
-    	elog(LOG, "YARN mode resource broker tries to return container of id "
-    			  INT64_FORMAT,
-    			  containerids[i]);
+    	elog(RMLOG, "YARN mode resource broker tries to return container of id "
+    			    INT64_FORMAT,
+					containerids[i]);
     }
 
 	if ( YARNJobID == NULL )
@@ -1169,9 +1169,9 @@ int handleRM2RB_GetContainerReport(void)
 
     for( int i = 0 ; i < size ; ++i )
     {
-    	elog(LOG, "Container report ID:"INT64_FORMAT", isActive:%d",
-    			  ctnstats[i].ContainerID,
-				  ctnstats[i].isActive);
+    	elog(RMLOG, "Container report ID:"INT64_FORMAT", isActive:%d",
+    			    ctnstats[i].ContainerID,
+					ctnstats[i].isActive);
     }
 
 	/* Build response message. */
@@ -1396,9 +1396,9 @@ int RB2YARN_getClusterReport(DQueue hosts)
 							  &nodeReportArraySize);
     if ( yarnres != FUNCTION_SUCCEEDED )
     {
-    	elog(LOG, "YARN mode resource broker failed to get cluster information "
-    			  "from YARN. %s",
-				  getErrorMessage());
+    	elog(WARNING, "YARN mode resource broker failed to get cluster "
+    				  "information from YARN. %s",
+					  getErrorMessage());
     }
     else
     {
@@ -1525,37 +1525,37 @@ int RB2YARN_getClusterReport(DQueue hosts)
 
     		insertDQueueTailNode(hosts, segstat);
 
-        	elog(LOG, "YARN mode resource broker got YARN cluster host \n"
-        			  "\thost:%s\n"
-					  "\tport:%d\n"
-					  "\thttpAddress:%s\n"
-        			  "\trackName:%s\n"
-        			  "\tmemoryUsed:%d\n"
-        			  "\tvcoresUsed:%d\n"
-        			  "\tmemoryCapability:%d\n"
-        			  "\tvcoresCapability:%d\n"
-        			  "\tnumContainers:%d\n"
-        			  "\tnodeState:%d\n"
-        			  "\thealthReport:%s\n"
-        			  "\tlastHealthReportTime:"INT64_FORMAT"\n"
-					  "\tmachineidsize:%d",
-    				  pnodereport->host,
-					  pnodereport->port,
-					  pnodereport->httpAddress,
-					  pnodereport->rackName,
-					  pnodereport->memoryUsed,
-					  pnodereport->vcoresUsed,
-					  pnodereport->memoryCapability,
-					  pnodereport->vcoresCapability,
-					  pnodereport->numContainers,
-					  pnodereport->nodeState,
-					  pnodereport->healthReport,
-					  pnodereport->lastHealthReportTime,
-					  segstat->Info.Size);
-
-        	elog(LOG, "YARN mode reosurce broker built cluster segment %s at rack %s",
-        			  GET_SEGINFO_GRMHOSTNAME(&(segstat->Info)),
-        			  GET_SEGINFO_GRMRACKNAME(&(segstat->Info)));
+        	elog(RMLOG, "YARN mode resource broker got YARN cluster host \n"
+        			    "\thost:%s\n"
+					    "\tport:%d\n"
+					    "\thttpAddress:%s\n"
+        			    "\trackName:%s\n"
+        			    "\tmemoryUsed:%d\n"
+        			    "\tvcoresUsed:%d\n"
+        			    "\tmemoryCapability:%d\n"
+        			    "\tvcoresCapability:%d\n"
+        			    "\tnumContainers:%d\n"
+        			    "\tnodeState:%d\n"
+        			    "\thealthReport:%s\n"
+        			    "\tlastHealthReportTime:"INT64_FORMAT"\n"
+					    "\tmachineidsize:%d",
+    				    pnodereport->host,
+					    pnodereport->port,
+					    pnodereport->httpAddress,
+					    pnodereport->rackName,
+					    pnodereport->memoryUsed,
+					    pnodereport->vcoresUsed,
+					    pnodereport->memoryCapability,
+					    pnodereport->vcoresCapability,
+					    pnodereport->numContainers,
+					    pnodereport->nodeState,
+					    pnodereport->healthReport,
+					    pnodereport->lastHealthReportTime,
+					    segstat->Info.Size);
+
+        	elog(RMLOG, "YARN mode reosurce broker built cluster segment %s at rack %s",
+        			    GET_SEGINFO_GRMHOSTNAME(&(segstat->Info)),
+        			    GET_SEGINFO_GRMRACKNAME(&(segstat->Info)));
     	}
     	freeMemNodeReportArray(nodeReportArray, nodeReportArraySize);
     }
@@ -1592,9 +1592,9 @@ int RB2YARN_acquireResource(uint32_t memorymb,
 								&allocatedResourcesArray,
 								&allocatedResourcesArraySize);
     if( yarnres != FUNCTION_SUCCEEDED ) {
-    	elog(LOG, "YARN mode resource broker failed to allocate "
-    			  "containers from YARN. %s",
-				  getErrorMessage());
+    	elog(WARNING, "YARN mode resource broker failed to allocate "
+    			  	  "containers from YARN. %s",
+					  getErrorMessage());
     	return yarnres;
     }
     else if ( allocatedResourcesArraySize == 0 ) {
@@ -1640,17 +1640,18 @@ int RB2YARN_acquireResource(uint32_t memorymb,
     									&activeFailIds,
 										&activeFailSize);
     if( yarnres != FUNCTION_SUCCEEDED ) {
-    	elog(LOG, "YARN mode resource broker failed to get active-fail "
-    			  "containers. %s",
-    			  getErrorMessage());
+    	elog(WARNING, "YARN mode resource broker failed to get active-fail "
+    			  	  "containers. %s",
+					  getErrorMessage());
     	goto exit;
     }
 
     /* Build temporary failed container ids in hash table for fast retrieving.*/
     if ( activeFailSize > 0 ) {
     	for (int i = 0 ; i < activeFailSize ; ++i) {
-    		elog(LOG, "YARN mode resource broker failed to activate container "INT64_FORMAT,
-    				  activeFailIds[i]);
+    		elog(WARNING, "YARN mode resource broker failed to activate "
+    					  "container "INT64_FORMAT,
+						  activeFailIds[i]);
 
 			SimpArray key;
 			setSimpleArrayRef(&key, (void *)&(activeFailIds[i]), sizeof(int64_t));
@@ -1662,7 +1663,7 @@ int RB2YARN_acquireResource(uint32_t memorymb,
 								   activeFailIds,
 								   activeFailSize);
     	if ( yarnres != FUNCTION_SUCCEEDED ) {
-    		elog(LOG, "YARN mode resource broker failed to return active-fail "
+    		elog(WARNING, "YARN mode resource broker failed to return active-fail "
     					  "containers. %s",
 						  getErrorMessage());
     	}
@@ -1681,12 +1682,12 @@ int RB2YARN_acquireResource(uint32_t memorymb,
     	strcpy(hostnamestr, allocatedResourcesArray[i].host);
     	insertDQueueTailNode(containerhosts, hostnamestr);
 
-    	elog(LOG, "YARN mode resource broker allocated and activated container. "
-    			  "ID : "INT64_FORMAT"(%d MB, %d CORE) at %s.",
-				  allocatedResourcesArray[i].containerId,
-				  allocatedResourcesArray[i].memory,
-				  allocatedResourcesArray[i].vCores,
-				  allocatedResourcesArray[i].host);
+    	elog(RMLOG, "YARN mode resource broker allocated and activated container. "
+    			    "ID : "INT64_FORMAT"(%d MB, %d CORE) at %s.",
+					allocatedResourcesArray[i].containerId,
+					allocatedResourcesArray[i].memory,
+					allocatedResourcesArray[i].vCores,
+					allocatedResourcesArray[i].host);
     }
 
 exit:
@@ -1716,8 +1717,8 @@ int RB2YARN_returnResource(int64_t *contids, int contcount)
 							   contids,
 							   contcount);
 	if ( yarnres != FUNCTION_SUCCEEDED ) {
-		elog(LOG, "YARN mode resource broker failed to return containers. %s",
-				  getErrorMessage());
+		elog(WARNING, "YARN mode resource broker failed to return containers. %s",
+				  	  getErrorMessage());
 	}
 
 	for ( int i = 0 ; i < contcount ; ++i ) {
@@ -1744,8 +1745,9 @@ int RB2YARN_getContainerReport(RB_GRMContainerStat *ctnstats, int *size)
 	yarnres = getContainerReports(LIBYARNClient, YARNJobID, &ctnrparr, &arrsize);
 	if ( yarnres != FUNCTION_SUCCEEDED )
 	{
-		elog(LOG, "YARN mode resource broker failed to get container report. %s",
-				  getErrorMessage());
+		elog(WARNING, "YARN mode resource broker failed to get container "
+					  "report. %s",
+					  getErrorMessage());
 	}
 	else if ( arrsize > 0 )
 	{
@@ -1769,8 +1771,9 @@ int RB2YARN_getContainerReport(RB_GRMContainerStat *ctnstats, int *size)
 									   &ctnstatsize);
         if ( yarnres != FUNCTION_SUCCEEDED )
         {
-        	elog(LOG, "YARN mode resource broker failed to get container status. %s",
-        			  getErrorMessage());
+        	elog(WARNING, "YARN mode resource broker failed to get container "
+        				  "status. %s",
+        			  	  getErrorMessage());
         }
 
         rm_pfree(PCONTEXT, ctnidarr);
@@ -1834,10 +1837,10 @@ int  RB2YARN_getQueueReport(char 	*queuename,
 	Assert( queuename != NULL );
 	yarnres = getQueueInfo(LIBYARNClient, queuename, true, true, true, &queueInfo);
 	if ( yarnres != FUNCTION_SUCCEEDED ) {
-		elog(LOG, "YARN mode resource broker failed to get YARN queue report of"
-				  " queue %s. %s",
-				  queuename,
-				  RB2YARN_getErrorMessage());
+		elog(WARNING, "YARN mode resource broker failed to get YARN queue report "
+					  "of queue %s. %s",
+					  queuename,
+					  RB2YARN_getErrorMessage());
 		return yarnres;
 	}
 
@@ -1857,8 +1860,8 @@ int  RB2YARN_finishYARNApplication(void)
 	int yarnres = FUNCTION_SUCCEEDED;
     yarnres = finishJob(LIBYARNClient, YARNJobID, APPLICATION_SUCCEEDED);
     if (yarnres != FUNCTION_SUCCEEDED) {
-    	elog(LOG, "YARN mode resource broker failed to finish job in YARN. %s",
-    			  getErrorMessage());
+    	elog(WARNING, "YARN mode resource broker failed to finish job in YARN. %s",
+    			  	  getErrorMessage());
     }
     else {
     	elog(LOG, "YARN mode resource broker finished job in YARN.");

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/37839128/src/backend/resourcemanager/resourcemanager.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcemanager.c b/src/backend/resourcemanager/resourcemanager.c
index f460ba5..ccbf460 100644
--- a/src/backend/resourcemanager/resourcemanager.c
+++ b/src/backend/resourcemanager/resourcemanager.c
@@ -1259,10 +1259,10 @@ int	 loadUserPropertiesFromCatalog(List **users)
 	foreach(cell, *users)
 	{
 		KVProperty property = lfirst(cell);
-		elog(LOG, "Resource manager loaded role specifications from pg_authid : "
-				  "[%s]=[%s]",
-				  property->Key.Str,
-				  property->Val.Str);
+		elog(RMLOG, "Resource manager loaded role specifications from pg_authid : "
+				    "[%s]=[%s]",
+					property->Key.Str,
+					property->Val.Str);
 	}
 	elog(LOG, "Resource manager successfully loaded role specifications.");
 
@@ -1569,10 +1569,10 @@ int	 loadQueuePropertiesFromCatalog(List **queues)
 	foreach(cell, *queues)
 	{
 		KVProperty property = lfirst(cell);
-		elog(LOG, "Resource manger loaded resource queue specifications from "
-				  "pg_resqueue : [%s]=[%s]",
-				  property->Key.Str,
-				  property->Val.Str);
+		elog(RMLOG, "Resource manger loaded resource queue specifications from "
+				  	"pg_resqueue : [%s]=[%s]",
+					property->Key.Str,
+					property->Val.Str);
 	}
 	elog(LOG, "Resource manger successfully loaded resource queue specifications");
 
@@ -2947,9 +2947,9 @@ bool cleanedAllGRMContainers(void)
 	if ( PRESPOOL->AddPendingContainerCount > 0 ||
 		 PRESPOOL->RetPendingContainerCount > 0 )
 	{
-		elog(DEBUG3, "Pending GRM container count inc %d, dec %d.",
-					 PRESPOOL->AddPendingContainerCount,
-					 PRESPOOL->RetPendingContainerCount);
+		elog(RMLOG, "Pending GRM container count inc %d, dec %d.",
+					PRESPOOL->AddPendingContainerCount,
+					PRESPOOL->RetPendingContainerCount);
 	}
 	/* Condition 2. No on-the-fly GRM containers for increasing and decreasing.*/
 	return PRESPOOL->AddPendingContainerCount == 0 &&

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/37839128/src/backend/resourcemanager/resourcepool.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcepool.c b/src/backend/resourcemanager/resourcepool.c
index 9941226..1e744f4 100644
--- a/src/backend/resourcemanager/resourcepool.c
+++ b/src/backend/resourcemanager/resourcepool.c
@@ -987,7 +987,11 @@ int addHAWQSegWithSegStat(SegStat segstat, bool *capstatchanged)
 			Assert(false);
 		}
 
-		elog(LOG, "Resource manager tracked segment %d of host %s.", segid, hostname);
+		SelfMaintainBufferData logcontent;
+		initializeSelfMaintainBuffer(&logcontent, PCONTEXT);
+		appendSelfMaintainBuffer(&logcontent, "host ", sizeof("host ")-1);
+		appendSelfMaintainBuffer(&logcontent, hostname, strlen(hostname));
+		appendSelfMaintainBuffer(&logcontent, " ip address ", sizeof(" ip address"));
 
 		/* Index all HAWQ node's ip addresses.
 		 *
@@ -1012,11 +1016,23 @@ int addHAWQSegWithSegStat(SegStat segstat, bool *capstatchanged)
 								  TYPCONVERT(void *, segid),
 								  false /* There should be no old value. */);
 
+				appendSelfMaintainBuffer(&logcontent, ",", sizeof(",")-1);
+
+				appendSelfMaintainBuffer(&logcontent,
+										 hostaddrkey.Array,
+										 hostaddrkey.Len);
+
 				elog(LOG, "Resource manager tracked ip address '%.*s' for host '%s'",
 						  hostaddrkey.Len, hostaddrkey.Array,
 						  hostname);
 			}
 		}
+		appendSMBStr(&logcontent, "");
+
+		elog(LOG, "Resource manager tracked segment %d of %s.",
+				  segid,
+				  SMBUFF_CONTENT(&logcontent));
+		destroySelfMaintainBuffer(&logcontent);
 
 		/*
 		 * If in GRM mode, the new registered segment is marked
@@ -1541,11 +1557,11 @@ int updateHAWQSegWithGRMSegStat( SegStat segstat)
 
 	segres->Stat = newSegStat;
 
-	elog(LOG, "Resource manager update segment info, hostname:%s, "
-			  "with GRM hostname:%s, GRM rackname:%s",
-			  GET_SEGINFO_HOSTNAME(&(newSegStat->Info)),
-			  GET_SEGINFO_GRMHOSTNAME(&(newSegStat->Info)),
-			  GET_SEGINFO_GRMRACKNAME(&(newSegStat->Info)));
+	elog(RMLOG, "Resource manager update segment info, hostname:%s, "
+			    "with GRM hostname:%s, GRM rackname:%s",
+				GET_SEGINFO_HOSTNAME(&(newSegStat->Info)),
+				GET_SEGINFO_GRMHOSTNAME(&(newSegStat->Info)),
+				GET_SEGINFO_GRMRACKNAME(&(newSegStat->Info)));
 
 	elog(RMLOG, "After resource manager "
 				"updates segment info's GRM host name and rack name, "
@@ -1999,8 +2015,8 @@ int  addGRMContainerToToBeAccepted(GRMContainer ctn)
 
 	appendGRMContainerSetContainer(ctns, ctn);
 	PRESPOOL->AddPendingContainerCount++;
-	elog(LOG, "AddPendingContainerCount added 1, current value %d.",
-			  PRESPOOL->AddPendingContainerCount);
+	elog(RMLOG, "AddPendingContainerCount added 1, current value %d.",
+			  	PRESPOOL->AddPendingContainerCount);
 	return FUNC_RETURN_OK;
 }
 
@@ -2073,15 +2089,15 @@ void addGRMContainerToResPool(GRMContainer container)
 	reorderSegResourceAllocIndex(segresource, ratio);
 	reorderSegResourceCombinedWorkloadIndex(segresource);
 
-	elog(LOG, "Resource manager added resource container into resource pool "
-			  "(%d MB, %d CORE) at %s (%d:%.*s), still pending %d MB",
-			  container->MemoryMB,
-			  container->Core,
-			  container->HostName,
-			  segresource->Stat->ID,
-			  segresource->Stat->Info.HostNameLen,
-			  GET_SEGRESOURCE_HOSTNAME(segresource),
-			  segresource->IncPending.MemoryMB);
+	elog(RMLOG, "Resource manager added resource container into resource pool "
+			  	"(%d MB, %d CORE) at %s (%d:%.*s), still pending %d MB",
+				container->MemoryMB,
+				container->Core,
+				container->HostName,
+				segresource->Stat->ID,
+				segresource->Stat->Info.HostNameLen,
+				GET_SEGRESOURCE_HOSTNAME(segresource),
+				segresource->IncPending.MemoryMB);
 }
 
 void addGRMContainerToToBeKicked(GRMContainer ctn)
@@ -3059,10 +3075,10 @@ void addSegResourceAllocIndex(SegResource segres)
 			Assert(false);
 		}
 
-		elog(LOG, "Resource manager tracked host %s in allocated resource "
-				  "ordered  index for mem/core ratio %d MBPCORE.",
-				  GET_SEGRESOURCE_HOSTNAME(segres),
-				  ratio);
+		elog(RMLOG, "Resource manager tracked host %s in allocated resource "
+				    "ordered  index for mem/core ratio %d MBPCORE.",
+				    GET_SEGRESOURCE_HOSTNAME(segres),
+				    ratio);
 	}
 }
 
@@ -3077,8 +3093,8 @@ void addSegResourceCombinedWorkloadIndex(SegResource segres)
 		Assert(false);
 	}
 
-	elog(LOG, "Resource manager tracked host %s in io bytes workload.",
-			  GET_SEGRESOURCE_HOSTNAME(segres));
+	elog(RMLOG, "Resource manager tracked host %s in io bytes workload.",
+			  	GET_SEGRESOURCE_HOSTNAME(segres));
 }
 
 int reorderSegResourceAvailIndex(SegResource segres, uint32_t ratio)
@@ -3490,8 +3506,8 @@ void moveAllAcceptedGRMContainersToResPool(void)
 
 		addGRMContainerToResPool(ctn);
 		PRESPOOL->AddPendingContainerCount--;
-		elog(LOG, "AddPendingContainerCount minused 1, current value %d",
-				  PRESPOOL->AddPendingContainerCount);
+		elog(RMLOG, "AddPendingContainerCount minus 1, current value %d",
+				  	PRESPOOL->AddPendingContainerCount);
 		addNewResourceToResourceManager(ctn->MemoryMB, ctn->Core);
 		removePendingResourceRequestInRootQueue(ctn->MemoryMB, ctn->Core, true);
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/37839128/src/backend/resourcemanager/resqueuemanager.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resqueuemanager.c b/src/backend/resourcemanager/resqueuemanager.c
index 630a38a..e20ef1b 100644
--- a/src/backend/resourcemanager/resqueuemanager.c
+++ b/src/backend/resourcemanager/resqueuemanager.c
@@ -1897,9 +1897,9 @@ int acquireResourceFromResQueMgr(ConnectionTrack  conntrack,
 	int						res			= FUNC_RETURN_OK;
 	DynResourceQueueTrack	queuetrack	= conntrack->QueueTrack;
 
-	elog(LOG, "ConnID %d. Expect query resource for session "INT64_FORMAT,
-			  conntrack->ConnID,
-			  conntrack->SessionID);
+	elog(RMLOG, "ConnID %d. Expect query resource for session "INT64_FORMAT,
+			    conntrack->ConnID,
+			    conntrack->SessionID);
 
 	/* Call quota logic to make decision of resource for current query. */
 	res = computeQueryQuota(conntrack, errorbuf, errorbufsize);
@@ -3397,13 +3397,13 @@ int computeQueryQuota(ConnectionTrack conn, char *errorbuf, int errorbufsize)
 			return res;
 		}
 
-		elog(LOG, "ConnID %d. Expect query resource (%d MB, %lf CORE) x %d "
-				  "(MIN %d) after checking queue capacity.",
-				  conn->ConnID,
-				  conn->SegMemoryMB,
-				  conn->SegCore,
-				  conn->SegNum,
-				  conn->SegNumMin);
+		elog(RMLOG, "ConnID %d. Expect query resource (%d MB, %lf CORE) x %d "
+				    "(MIN %d) after checking queue capacity.",
+					conn->ConnID,
+					conn->SegMemoryMB,
+					conn->SegCore,
+					conn->SegNum,
+					conn->SegNumMin);
 
 		/*------------------------------------------------------------------
 		 * The following logic consider the actual resource requirement from
@@ -3427,15 +3427,15 @@ int computeQueryQuota(ConnectionTrack conn, char *errorbuf, int errorbufsize)
 			conn->SegNum	= conn->MaxSegCountFixed;
 		}
 
-		elog(LOG, "ConnID %d. Expect query resource (%d MB, %lf CORE) x %d "
-				  "(MIN %d) after checking query expectation %d (MIN %d).",
-				  conn->ConnID,
-				  conn->SegMemoryMB,
-				  conn->SegCore,
-				  conn->SegNum,
-				  conn->SegNumMin,
-				  conn->MaxSegCountFixed,
-				  conn->MinSegCountFixed);
+		elog(RMLOG, "ConnID %d. Expect query resource (%d MB, %lf CORE) x %d "
+				    "(MIN %d) after checking query expectation %d (MIN %d).",
+					conn->ConnID,
+					conn->SegMemoryMB,
+					conn->SegCore,
+					conn->SegNum,
+					conn->SegNumMin,
+					conn->MaxSegCountFixed,
+					conn->MinSegCountFixed);
 
 	}
 
@@ -5003,9 +5003,9 @@ int rebuildAllResourceQueueTrackDynamicStatusInShadow(List *quehavingshadow,
 		}
 		else
 		{
-			elog(LOG, "Resource manager passed rebuilding resource queue %s "
-					  "dynamic status in its shadow.",
-					  quetrack->QueueInfo->Name);
+			elog(RMLOG, "Resource manager passed rebuilding resource queue %s "
+					    "dynamic status in its shadow.",
+					    quetrack->QueueInfo->Name);
 		}
 
 		res = detectAndDealWithDeadLockInShadow(quetrack, queuechanged);
@@ -5020,13 +5020,14 @@ int rebuildAllResourceQueueTrackDynamicStatusInShadow(List *quehavingshadow,
 		}
 		else
 		{
-			elog(LOG, "Resource manager passed detecting deadlock issues in the "
-					  "shadow of resource queue %s",
-					  quetrack->QueueInfo->Name);
+			elog(RMLOG, "Resource manager passed detecting deadlock issues in the "
+					    "shadow of resource queue %s",
+					    quetrack->QueueInfo->Name);
 		}
 	}
 
-	elog(LOG, "Resource manager finished rebuilding resource queues' dynamic status");
+	elog(RMLOG, "Resource manager finished rebuilding resource queues' dynamic "
+				"status");
 	return FUNC_RETURN_OK;
 }
 
@@ -5037,8 +5038,8 @@ int rebuildResourceQueueTrackDynamicStatusInShadow(DynResourceQueueTrack
 quetra
 {
 	int res = FUNC_RETURN_OK;
 
-	elog(LOG, "Rebuild resource queue %s dynamic status in its shadow.",
-			  quetrack->QueueInfo->Name);
+	elog(RMLOG, "Rebuild resource queue %s dynamic status in its shadow.",
+			    quetrack->QueueInfo->Name);
 
 	DynResourceQueueTrack shadowtrack = quetrack->ShadowQueueTrack;
 	/* Get deadlock detector ready in the shadow instance. */
@@ -5139,8 +5140,9 @@ int rebuildResourceQueueTrackDynamicStatusInShadow(DynResourceQueueTrack
 quetra
 				 shadowtrack->DLDetector.InUseTotal.MemoryMB,
 				 shadowtrack->DLDetector.LockedTotal.MemoryMB);
 
-	elog(LOG, "Finished rebuilding resource queue %s dynamic status in its shadow.",
-			  quetrack->QueueInfo->Name);
+	elog(RMLOG, "Finished rebuilding resource queue %s dynamic status in its "
+				"shadow.",
+			    quetrack->QueueInfo->Name);
 
 	return FUNC_RETURN_OK;
 }

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/37839128/src/backend/tcop/pquery.c
----------------------------------------------------------------------
diff --git a/src/backend/tcop/pquery.c b/src/backend/tcop/pquery.c
index 8ef445a..efa5bce 100644
--- a/src/backend/tcop/pquery.c
+++ b/src/backend/tcop/pquery.c
@@ -1123,8 +1123,6 @@ CleanupGlobalQueryResources(void)
 	int ret;
 	char errorbuf[1024];
 
-	elog(LOG, "In CleanupGlobalQueryResources().");
-
 	/* Force using new socket connection to return and free. */
 	bool oldval = rm_enable_connpool;
 	rm_enable_connpool = false;


Mime
View raw message