Repository: incubator-hawq Updated Branches: refs/heads/master 143c0e3f1 -> 37839128b
HAWQ-645. shrink LOG level log output from resource manager Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/37839128 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/37839128 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/37839128 Branch: refs/heads/master Commit: 37839128bfe8b063918300a929825485c1d0b3af Parents: 143c0e3 Author: YI JIN <y...@pivotal.io> Authored: Sat Apr 9 12:36:12 2016 +1000 Committer: YI JIN <y...@pivotal.io> Committed: Sat Apr 9 12:36:12 2016 +1000 ---------------------------------------------------------------------- .../communication/rmcomm_Message.c | 10 +- .../communication/rmcomm_QD2RM.c | 20 +-- src/backend/resourcemanager/requesthandler.c | 4 +- .../resourcebroker/resourcebroker_LIBYARN.c | 67 ++++++--- .../resourcebroker_LIBYARN_proc.c | 149 ++++++++++--------- src/backend/resourcemanager/resourcemanager.c | 22 +-- src/backend/resourcemanager/resourcepool.c | 66 ++++---- src/backend/resourcemanager/resqueuemanager.c | 62 ++++---- src/backend/tcop/pquery.c | 2 - 9 files changed, 224 insertions(+), 178 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/37839128/src/backend/resourcemanager/communication/rmcomm_Message.c ---------------------------------------------------------------------- diff --git a/src/backend/resourcemanager/communication/rmcomm_Message.c b/src/backend/resourcemanager/communication/rmcomm_Message.c index 364c8a2..b12a734 100644 --- a/src/backend/resourcemanager/communication/rmcomm_Message.c +++ b/src/backend/resourcemanager/communication/rmcomm_Message.c @@ -96,13 +96,9 @@ void ReadPostHandler_Message(AsyncCommBuffer buffer) if ( DRM_MSGFRAME_TAILTAG_MATCHED(p) ) { /* Skip heart-beat message log. */ - elog((header->MessageID == REQUEST_QD_REFRESH_RESOURCE || - header->MessageID == REQUEST_RM_IMALIVE || - header->MessageID == RESPONSE_QD_REFRESH_RESOURCE || - header->MessageID == RESPONSE_RM_IMALIVE) ? DEBUG3 : LOG, - "AsyncComm framework receives message %d from FD %d", - header->MessageID, - buffer->FD); + elog(RMLOG, "AsyncComm framework receives message %d from FD %d", + header->MessageID, + buffer->FD); /* Get complete message and call the handler. */ if ( context->MessageRecvedHandler != NULL ) http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/37839128/src/backend/resourcemanager/communication/rmcomm_QD2RM.c ---------------------------------------------------------------------- diff --git a/src/backend/resourcemanager/communication/rmcomm_QD2RM.c b/src/backend/resourcemanager/communication/rmcomm_QD2RM.c index 4808a8b..f5cd379 100644 --- a/src/backend/resourcemanager/communication/rmcomm_QD2RM.c +++ b/src/backend/resourcemanager/communication/rmcomm_QD2RM.c @@ -698,16 +698,16 @@ int acquireResourceFromRM(int index, 0 : preferred_nodes_size; - elog(LOG, "ConnID: %d. Acquire resource request for index %d. " - "Max vseg size %d Min vseg size %d Estimated slice size %d " - "estimated IO bytes size " INT64_FORMAT " Preferred node count %d.", - curcontext->QD_Conn_ID, - index, - max_seg_count_fix, - min_seg_count_fix, - slice_size, - iobytes, - nodecount); + elog(RMLOG, "ConnID: %d. Acquire resource request for index %d. " + "Max vseg size %d Min vseg size %d Estimated slice size %d " + "estimated IO bytes size " INT64_FORMAT " Preferred node count %d.", + curcontext->QD_Conn_ID, + index, + max_seg_count_fix, + min_seg_count_fix, + slice_size, + iobytes, + nodecount); /* Build request. */ resetSelfMaintainBuffer(sendbuffer); http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/37839128/src/backend/resourcemanager/requesthandler.c ---------------------------------------------------------------------- diff --git a/src/backend/resourcemanager/requesthandler.c b/src/backend/resourcemanager/requesthandler.c index daef075..eb9fc1d 100644 --- a/src/backend/resourcemanager/requesthandler.c +++ b/src/backend/resourcemanager/requesthandler.c @@ -178,8 +178,8 @@ bool handleRMRequestConnectionRegByOID(void **arg) if ( res == FUNC_RETURN_OK ) { trackConnectionTrack(conntrack); - elog(LOG, "ConnID %d. Resource manager tracked connection.", - conntrack->ConnID); + elog(RMLOG, "ConnID %d. Resource manager tracked connection.", + conntrack->ConnID); response.Result = FUNC_RETURN_OK; response.ConnID = conntrack->ConnID; } http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/37839128/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c ---------------------------------------------------------------------- diff --git a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c index b8d205e..37e3bca 100644 --- a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c +++ b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c @@ -454,8 +454,8 @@ int RB_LIBYARN_getContainerReport(List **ctnstatl) res = RESBROK_PIPE_ERROR; } destroySelfMaintainBuffer(&sendBuffer); - elog(LOG, "YARN mode resource broker wrote get container status request to " - "resource broker process."); + elog(RMLOG, "YARN mode resource broker wrote get container status request to " + "resource broker process."); PipeReceivePending = res == FUNC_RETURN_OK; return res; } @@ -509,7 +509,7 @@ int RB_LIBYARN_handleNotification(void) } PipeReceivePending = false; - elog(LOG, "Finish processing message %d", messageid); + elog(RMLOG, "Finish processing message %d", messageid); if ( res != FUNC_RETURN_OK ) { @@ -568,9 +568,11 @@ int handleRB2RM_ClusterReport(void) } elog(LOG, "YARN mode resource broker got cluster report having %d host(s), " - "maximum queue capacity is %lf.", + "queue capacity %lf (MAX %lf) current capacity %lf.", response.MachineCount, - response.QueueMaxCapacity); + response.QueueCapacity, + response.QueueMaxCapacity, + response.QueueCurCapacity); /* Read machines. */ for( int i = 0 ; i < response.MachineCount ; ++i ) @@ -659,12 +661,20 @@ int handleRB2RM_ClusterReport(void) setAllSegResourceGRMUnhandled(); + + ResourceBundleData oldGRMTotalAll; + ResourceBundleData oldGRMTotal; + resetResourceBundleDataByBundle(&oldGRMTotalAll, + &(PRESPOOL->GRMTotalHavingNoHAWQNode)); + resetResourceBundleDataByBundle(&oldGRMTotal, + &(PRESPOOL->GRMTotal)); /* * Start to update resource pool content. The YARN cluster total size is * also counted the same time. */ - resetResourceBundleData(&(PRESPOOL->GRMTotalHavingNoHAWQNode), 0, 0.0, 0); + int updatedcnt = 0; + int skipedcnt = 0; MEMORY_CONTEXT_SWITCH_TO(PCONTEXT) while( list_length(segstats) > 0 ) @@ -681,20 +691,28 @@ int handleRB2RM_ClusterReport(void) SelfMaintainBufferData buffer; initializeSelfMaintainBuffer(&buffer, PCONTEXT); generateSegStatReport(segstat, &buffer); - elog(LOG, "YARN mode resource broker updated segment configuration " - "in resource pool. %s", buffer.Buffer); + elog(RMLOG, "YARN mode resource broker updated segment configuration " + "in resource pool. %s", buffer.Buffer); destroySelfMaintainBuffer(&buffer); + updatedcnt++; } else { - elog(LOG, "YARN mode resource broker skipped segment configuration " - "from host %s", - GET_SEGINFO_GRMHOSTNAME(&(segstat->Info))); + elog(WARNING, "YARN mode resource broker skipped segment configuration " + "from host %s", + GET_SEGINFO_GRMHOSTNAME(&(segstat->Info))); + skipedcnt++; } rm_pfree(PCONTEXT, segstat); segstats = list_delete_first(segstats); } + elog(LOG, "YARN mode resource broker updated %d %s, skiped %d %s", + updatedcnt, + updatedcnt <= 1 ? "segment" : "segments", + skipedcnt, + skipedcnt <= 1 ? "segment" : "segments"); + /* * iterate all segments without GRM report, * and update its status. @@ -754,13 +772,26 @@ int handleRB2RM_ClusterReport(void) MEMORY_CONTEXT_SWITCH_BACK - elog(LOG, "Resource manager YARN resource broker counted HAWQ cluster now " - "having (%d MB, %lf CORE) in a YARN cluster of total resource " - "(%d MB, %lf CORE).", - PRESPOOL->GRMTotal.MemoryMB, - PRESPOOL->GRMTotal.Core, - PRESPOOL->GRMTotalHavingNoHAWQNode.MemoryMB, - PRESPOOL->GRMTotalHavingNoHAWQNode.Core); + if ( oldGRMTotalAll.MemoryMB != PRESPOOL->GRMTotalHavingNoHAWQNode.MemoryMB || + oldGRMTotalAll.Core != PRESPOOL->GRMTotalHavingNoHAWQNode.Core ) + { + elog(LOG, "Resource manager YARN resource broker counted YARN cluster " + "having total resource (%d MB, %lf CORE).", + PRESPOOL->GRMTotalHavingNoHAWQNode.MemoryMB, + PRESPOOL->GRMTotalHavingNoHAWQNode.Core); + } + + if ( oldGRMTotal.MemoryMB != PRESPOOL->GRMTotal.MemoryMB || + oldGRMTotal.Core != PRESPOOL->GRMTotal.Core ) + { + elog(LOG, "Resource manager YARN resource broker counted HAWQ cluster now " + "having (%d MB, %lf CORE) in a YARN cluster of total resource " + "(%d MB, %lf CORE).", + PRESPOOL->GRMTotal.MemoryMB, + PRESPOOL->GRMTotal.Core, + PRESPOOL->GRMTotalHavingNoHAWQNode.MemoryMB, + PRESPOOL->GRMTotalHavingNoHAWQNode.Core); + } /* * If the segment is GRM unavailable or FTS unavailable, http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/37839128/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN_proc.c ---------------------------------------------------------------------- diff --git a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN_proc.c b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN_proc.c index ad49aae..5a6e6a2 100644 --- a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN_proc.c +++ b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN_proc.c @@ -189,8 +189,8 @@ int ResBrokerMainInternal(void) * run anymore. The process goes to the exit phase directly. */ if ( getppid() != ResBrokerParentPID ) { - elog(LOG, "Parent process of YARN mode resource broker quited. " - "Resource broker process will actively close."); + elog(WARNING, "Parent process of YARN mode resource broker quit. " + "Resource broker process will actively close."); ResBrokerKeepRun = false; continue; } @@ -270,10 +270,10 @@ int ResBrokerMainInternal(void) } if ( res != FUNC_RETURN_OK ) { - elog(LOG, "YARN mode resource broker failed to process request. " - "Message id = %d, result = %d.", - messageid, - res); + elog(WARNING, "YARN mode resource broker failed to process request. " + "Message id = %d, result = %d.", + messageid, + res); /* If this is a pipe error between RM and RB or YARN remove error. * Exit RB and let RM restart RB process. */ if ( res == RESBROK_ERROR_GRM ) @@ -1035,9 +1035,9 @@ int handleRM2RB_ReturnResource(void) for ( int i = 0 ; i < request.ContainerCount ; ++i ) { - elog(LOG, "YARN mode resource broker tries to return container of id " - INT64_FORMAT, - containerids[i]); + elog(RMLOG, "YARN mode resource broker tries to return container of id " + INT64_FORMAT, + containerids[i]); } if ( YARNJobID == NULL ) @@ -1169,9 +1169,9 @@ int handleRM2RB_GetContainerReport(void) for( int i = 0 ; i < size ; ++i ) { - elog(LOG, "Container report ID:"INT64_FORMAT", isActive:%d", - ctnstats[i].ContainerID, - ctnstats[i].isActive); + elog(RMLOG, "Container report ID:"INT64_FORMAT", isActive:%d", + ctnstats[i].ContainerID, + ctnstats[i].isActive); } /* Build response message. */ @@ -1396,9 +1396,9 @@ int RB2YARN_getClusterReport(DQueue hosts) &nodeReportArraySize); if ( yarnres != FUNCTION_SUCCEEDED ) { - elog(LOG, "YARN mode resource broker failed to get cluster information " - "from YARN. %s", - getErrorMessage()); + elog(WARNING, "YARN mode resource broker failed to get cluster " + "information from YARN. %s", + getErrorMessage()); } else { @@ -1525,37 +1525,37 @@ int RB2YARN_getClusterReport(DQueue hosts) insertDQueueTailNode(hosts, segstat); - elog(LOG, "YARN mode resource broker got YARN cluster host \n" - "\thost:%s\n" - "\tport:%d\n" - "\thttpAddress:%s\n" - "\trackName:%s\n" - "\tmemoryUsed:%d\n" - "\tvcoresUsed:%d\n" - "\tmemoryCapability:%d\n" - "\tvcoresCapability:%d\n" - "\tnumContainers:%d\n" - "\tnodeState:%d\n" - "\thealthReport:%s\n" - "\tlastHealthReportTime:"INT64_FORMAT"\n" - "\tmachineidsize:%d", - pnodereport->host, - pnodereport->port, - pnodereport->httpAddress, - pnodereport->rackName, - pnodereport->memoryUsed, - pnodereport->vcoresUsed, - pnodereport->memoryCapability, - pnodereport->vcoresCapability, - pnodereport->numContainers, - pnodereport->nodeState, - pnodereport->healthReport, - pnodereport->lastHealthReportTime, - segstat->Info.Size); - - elog(LOG, "YARN mode reosurce broker built cluster segment %s at rack %s", - GET_SEGINFO_GRMHOSTNAME(&(segstat->Info)), - GET_SEGINFO_GRMRACKNAME(&(segstat->Info))); + elog(RMLOG, "YARN mode resource broker got YARN cluster host \n" + "\thost:%s\n" + "\tport:%d\n" + "\thttpAddress:%s\n" + "\trackName:%s\n" + "\tmemoryUsed:%d\n" + "\tvcoresUsed:%d\n" + "\tmemoryCapability:%d\n" + "\tvcoresCapability:%d\n" + "\tnumContainers:%d\n" + "\tnodeState:%d\n" + "\thealthReport:%s\n" + "\tlastHealthReportTime:"INT64_FORMAT"\n" + "\tmachineidsize:%d", + pnodereport->host, + pnodereport->port, + pnodereport->httpAddress, + pnodereport->rackName, + pnodereport->memoryUsed, + pnodereport->vcoresUsed, + pnodereport->memoryCapability, + pnodereport->vcoresCapability, + pnodereport->numContainers, + pnodereport->nodeState, + pnodereport->healthReport, + pnodereport->lastHealthReportTime, + segstat->Info.Size); + + elog(RMLOG, "YARN mode reosurce broker built cluster segment %s at rack %s", + GET_SEGINFO_GRMHOSTNAME(&(segstat->Info)), + GET_SEGINFO_GRMRACKNAME(&(segstat->Info))); } freeMemNodeReportArray(nodeReportArray, nodeReportArraySize); } @@ -1592,9 +1592,9 @@ int RB2YARN_acquireResource(uint32_t memorymb, &allocatedResourcesArray, &allocatedResourcesArraySize); if( yarnres != FUNCTION_SUCCEEDED ) { - elog(LOG, "YARN mode resource broker failed to allocate " - "containers from YARN. %s", - getErrorMessage()); + elog(WARNING, "YARN mode resource broker failed to allocate " + "containers from YARN. %s", + getErrorMessage()); return yarnres; } else if ( allocatedResourcesArraySize == 0 ) { @@ -1640,17 +1640,18 @@ int RB2YARN_acquireResource(uint32_t memorymb, &activeFailIds, &activeFailSize); if( yarnres != FUNCTION_SUCCEEDED ) { - elog(LOG, "YARN mode resource broker failed to get active-fail " - "containers. %s", - getErrorMessage()); + elog(WARNING, "YARN mode resource broker failed to get active-fail " + "containers. %s", + getErrorMessage()); goto exit; } /* Build temporary failed container ids in hash table for fast retrieving.*/ if ( activeFailSize > 0 ) { for (int i = 0 ; i < activeFailSize ; ++i) { - elog(LOG, "YARN mode resource broker failed to activate container "INT64_FORMAT, - activeFailIds[i]); + elog(WARNING, "YARN mode resource broker failed to activate " + "container "INT64_FORMAT, + activeFailIds[i]); SimpArray key; setSimpleArrayRef(&key, (void *)&(activeFailIds[i]), sizeof(int64_t)); @@ -1662,7 +1663,7 @@ int RB2YARN_acquireResource(uint32_t memorymb, activeFailIds, activeFailSize); if ( yarnres != FUNCTION_SUCCEEDED ) { - elog(LOG, "YARN mode resource broker failed to return active-fail " + elog(WARNING, "YARN mode resource broker failed to return active-fail " "containers. %s", getErrorMessage()); } @@ -1681,12 +1682,12 @@ int RB2YARN_acquireResource(uint32_t memorymb, strcpy(hostnamestr, allocatedResourcesArray[i].host); insertDQueueTailNode(containerhosts, hostnamestr); - elog(LOG, "YARN mode resource broker allocated and activated container. " - "ID : "INT64_FORMAT"(%d MB, %d CORE) at %s.", - allocatedResourcesArray[i].containerId, - allocatedResourcesArray[i].memory, - allocatedResourcesArray[i].vCores, - allocatedResourcesArray[i].host); + elog(RMLOG, "YARN mode resource broker allocated and activated container. " + "ID : "INT64_FORMAT"(%d MB, %d CORE) at %s.", + allocatedResourcesArray[i].containerId, + allocatedResourcesArray[i].memory, + allocatedResourcesArray[i].vCores, + allocatedResourcesArray[i].host); } exit: @@ -1716,8 +1717,8 @@ int RB2YARN_returnResource(int64_t *contids, int contcount) contids, contcount); if ( yarnres != FUNCTION_SUCCEEDED ) { - elog(LOG, "YARN mode resource broker failed to return containers. %s", - getErrorMessage()); + elog(WARNING, "YARN mode resource broker failed to return containers. %s", + getErrorMessage()); } for ( int i = 0 ; i < contcount ; ++i ) { @@ -1744,8 +1745,9 @@ int RB2YARN_getContainerReport(RB_GRMContainerStat *ctnstats, int *size) yarnres = getContainerReports(LIBYARNClient, YARNJobID, &ctnrparr, &arrsize); if ( yarnres != FUNCTION_SUCCEEDED ) { - elog(LOG, "YARN mode resource broker failed to get container report. %s", - getErrorMessage()); + elog(WARNING, "YARN mode resource broker failed to get container " + "report. %s", + getErrorMessage()); } else if ( arrsize > 0 ) { @@ -1769,8 +1771,9 @@ int RB2YARN_getContainerReport(RB_GRMContainerStat *ctnstats, int *size) &ctnstatsize); if ( yarnres != FUNCTION_SUCCEEDED ) { - elog(LOG, "YARN mode resource broker failed to get container status. %s", - getErrorMessage()); + elog(WARNING, "YARN mode resource broker failed to get container " + "status. %s", + getErrorMessage()); } rm_pfree(PCONTEXT, ctnidarr); @@ -1834,10 +1837,10 @@ int RB2YARN_getQueueReport(char *queuename, Assert( queuename != NULL ); yarnres = getQueueInfo(LIBYARNClient, queuename, true, true, true, &queueInfo); if ( yarnres != FUNCTION_SUCCEEDED ) { - elog(LOG, "YARN mode resource broker failed to get YARN queue report of" - " queue %s. %s", - queuename, - RB2YARN_getErrorMessage()); + elog(WARNING, "YARN mode resource broker failed to get YARN queue report " + "of queue %s. %s", + queuename, + RB2YARN_getErrorMessage()); return yarnres; } @@ -1857,8 +1860,8 @@ int RB2YARN_finishYARNApplication(void) int yarnres = FUNCTION_SUCCEEDED; yarnres = finishJob(LIBYARNClient, YARNJobID, APPLICATION_SUCCEEDED); if (yarnres != FUNCTION_SUCCEEDED) { - elog(LOG, "YARN mode resource broker failed to finish job in YARN. %s", - getErrorMessage()); + elog(WARNING, "YARN mode resource broker failed to finish job in YARN. %s", + getErrorMessage()); } else { elog(LOG, "YARN mode resource broker finished job in YARN."); http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/37839128/src/backend/resourcemanager/resourcemanager.c ---------------------------------------------------------------------- diff --git a/src/backend/resourcemanager/resourcemanager.c b/src/backend/resourcemanager/resourcemanager.c index f460ba5..ccbf460 100644 --- a/src/backend/resourcemanager/resourcemanager.c +++ b/src/backend/resourcemanager/resourcemanager.c @@ -1259,10 +1259,10 @@ int loadUserPropertiesFromCatalog(List **users) foreach(cell, *users) { KVProperty property = lfirst(cell); - elog(LOG, "Resource manager loaded role specifications from pg_authid : " - "[%s]=[%s]", - property->Key.Str, - property->Val.Str); + elog(RMLOG, "Resource manager loaded role specifications from pg_authid : " + "[%s]=[%s]", + property->Key.Str, + property->Val.Str); } elog(LOG, "Resource manager successfully loaded role specifications."); @@ -1569,10 +1569,10 @@ int loadQueuePropertiesFromCatalog(List **queues) foreach(cell, *queues) { KVProperty property = lfirst(cell); - elog(LOG, "Resource manger loaded resource queue specifications from " - "pg_resqueue : [%s]=[%s]", - property->Key.Str, - property->Val.Str); + elog(RMLOG, "Resource manger loaded resource queue specifications from " + "pg_resqueue : [%s]=[%s]", + property->Key.Str, + property->Val.Str); } elog(LOG, "Resource manger successfully loaded resource queue specifications"); @@ -2947,9 +2947,9 @@ bool cleanedAllGRMContainers(void) if ( PRESPOOL->AddPendingContainerCount > 0 || PRESPOOL->RetPendingContainerCount > 0 ) { - elog(DEBUG3, "Pending GRM container count inc %d, dec %d.", - PRESPOOL->AddPendingContainerCount, - PRESPOOL->RetPendingContainerCount); + elog(RMLOG, "Pending GRM container count inc %d, dec %d.", + PRESPOOL->AddPendingContainerCount, + PRESPOOL->RetPendingContainerCount); } /* Condition 2. No on-the-fly GRM containers for increasing and decreasing.*/ return PRESPOOL->AddPendingContainerCount == 0 && http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/37839128/src/backend/resourcemanager/resourcepool.c ---------------------------------------------------------------------- diff --git a/src/backend/resourcemanager/resourcepool.c b/src/backend/resourcemanager/resourcepool.c index 9941226..1e744f4 100644 --- a/src/backend/resourcemanager/resourcepool.c +++ b/src/backend/resourcemanager/resourcepool.c @@ -987,7 +987,11 @@ int addHAWQSegWithSegStat(SegStat segstat, bool *capstatchanged) Assert(false); } - elog(LOG, "Resource manager tracked segment %d of host %s.", segid, hostname); + SelfMaintainBufferData logcontent; + initializeSelfMaintainBuffer(&logcontent, PCONTEXT); + appendSelfMaintainBuffer(&logcontent, "host ", sizeof("host ")-1); + appendSelfMaintainBuffer(&logcontent, hostname, strlen(hostname)); + appendSelfMaintainBuffer(&logcontent, " ip address ", sizeof(" ip address")); /* Index all HAWQ node's ip addresses. * @@ -1012,11 +1016,23 @@ int addHAWQSegWithSegStat(SegStat segstat, bool *capstatchanged) TYPCONVERT(void *, segid), false /* There should be no old value. */); + appendSelfMaintainBuffer(&logcontent, ",", sizeof(",")-1); + + appendSelfMaintainBuffer(&logcontent, + hostaddrkey.Array, + hostaddrkey.Len); + elog(LOG, "Resource manager tracked ip address '%.*s' for host '%s'", hostaddrkey.Len, hostaddrkey.Array, hostname); } } + appendSMBStr(&logcontent, ""); + + elog(LOG, "Resource manager tracked segment %d of %s.", + segid, + SMBUFF_CONTENT(&logcontent)); + destroySelfMaintainBuffer(&logcontent); /* * If in GRM mode, the new registered segment is marked @@ -1541,11 +1557,11 @@ int updateHAWQSegWithGRMSegStat( SegStat segstat) segres->Stat = newSegStat; - elog(LOG, "Resource manager update segment info, hostname:%s, " - "with GRM hostname:%s, GRM rackname:%s", - GET_SEGINFO_HOSTNAME(&(newSegStat->Info)), - GET_SEGINFO_GRMHOSTNAME(&(newSegStat->Info)), - GET_SEGINFO_GRMRACKNAME(&(newSegStat->Info))); + elog(RMLOG, "Resource manager update segment info, hostname:%s, " + "with GRM hostname:%s, GRM rackname:%s", + GET_SEGINFO_HOSTNAME(&(newSegStat->Info)), + GET_SEGINFO_GRMHOSTNAME(&(newSegStat->Info)), + GET_SEGINFO_GRMRACKNAME(&(newSegStat->Info))); elog(RMLOG, "After resource manager " "updates segment info's GRM host name and rack name, " @@ -1999,8 +2015,8 @@ int addGRMContainerToToBeAccepted(GRMContainer ctn) appendGRMContainerSetContainer(ctns, ctn); PRESPOOL->AddPendingContainerCount++; - elog(LOG, "AddPendingContainerCount added 1, current value %d.", - PRESPOOL->AddPendingContainerCount); + elog(RMLOG, "AddPendingContainerCount added 1, current value %d.", + PRESPOOL->AddPendingContainerCount); return FUNC_RETURN_OK; } @@ -2073,15 +2089,15 @@ void addGRMContainerToResPool(GRMContainer container) reorderSegResourceAllocIndex(segresource, ratio); reorderSegResourceCombinedWorkloadIndex(segresource); - elog(LOG, "Resource manager added resource container into resource pool " - "(%d MB, %d CORE) at %s (%d:%.*s), still pending %d MB", - container->MemoryMB, - container->Core, - container->HostName, - segresource->Stat->ID, - segresource->Stat->Info.HostNameLen, - GET_SEGRESOURCE_HOSTNAME(segresource), - segresource->IncPending.MemoryMB); + elog(RMLOG, "Resource manager added resource container into resource pool " + "(%d MB, %d CORE) at %s (%d:%.*s), still pending %d MB", + container->MemoryMB, + container->Core, + container->HostName, + segresource->Stat->ID, + segresource->Stat->Info.HostNameLen, + GET_SEGRESOURCE_HOSTNAME(segresource), + segresource->IncPending.MemoryMB); } void addGRMContainerToToBeKicked(GRMContainer ctn) @@ -3059,10 +3075,10 @@ void addSegResourceAllocIndex(SegResource segres) Assert(false); } - elog(LOG, "Resource manager tracked host %s in allocated resource " - "ordered index for mem/core ratio %d MBPCORE.", - GET_SEGRESOURCE_HOSTNAME(segres), - ratio); + elog(RMLOG, "Resource manager tracked host %s in allocated resource " + "ordered index for mem/core ratio %d MBPCORE.", + GET_SEGRESOURCE_HOSTNAME(segres), + ratio); } } @@ -3077,8 +3093,8 @@ void addSegResourceCombinedWorkloadIndex(SegResource segres) Assert(false); } - elog(LOG, "Resource manager tracked host %s in io bytes workload.", - GET_SEGRESOURCE_HOSTNAME(segres)); + elog(RMLOG, "Resource manager tracked host %s in io bytes workload.", + GET_SEGRESOURCE_HOSTNAME(segres)); } int reorderSegResourceAvailIndex(SegResource segres, uint32_t ratio) @@ -3490,8 +3506,8 @@ void moveAllAcceptedGRMContainersToResPool(void) addGRMContainerToResPool(ctn); PRESPOOL->AddPendingContainerCount--; - elog(LOG, "AddPendingContainerCount minused 1, current value %d", - PRESPOOL->AddPendingContainerCount); + elog(RMLOG, "AddPendingContainerCount minus 1, current value %d", + PRESPOOL->AddPendingContainerCount); addNewResourceToResourceManager(ctn->MemoryMB, ctn->Core); removePendingResourceRequestInRootQueue(ctn->MemoryMB, ctn->Core, true); http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/37839128/src/backend/resourcemanager/resqueuemanager.c ---------------------------------------------------------------------- diff --git a/src/backend/resourcemanager/resqueuemanager.c b/src/backend/resourcemanager/resqueuemanager.c index 630a38a..e20ef1b 100644 --- a/src/backend/resourcemanager/resqueuemanager.c +++ b/src/backend/resourcemanager/resqueuemanager.c @@ -1897,9 +1897,9 @@ int acquireResourceFromResQueMgr(ConnectionTrack conntrack, int res = FUNC_RETURN_OK; DynResourceQueueTrack queuetrack = conntrack->QueueTrack; - elog(LOG, "ConnID %d. Expect query resource for session "INT64_FORMAT, - conntrack->ConnID, - conntrack->SessionID); + elog(RMLOG, "ConnID %d. Expect query resource for session "INT64_FORMAT, + conntrack->ConnID, + conntrack->SessionID); /* Call quota logic to make decision of resource for current query. */ res = computeQueryQuota(conntrack, errorbuf, errorbufsize); @@ -3397,13 +3397,13 @@ int computeQueryQuota(ConnectionTrack conn, char *errorbuf, int errorbufsize) return res; } - elog(LOG, "ConnID %d. Expect query resource (%d MB, %lf CORE) x %d " - "(MIN %d) after checking queue capacity.", - conn->ConnID, - conn->SegMemoryMB, - conn->SegCore, - conn->SegNum, - conn->SegNumMin); + elog(RMLOG, "ConnID %d. Expect query resource (%d MB, %lf CORE) x %d " + "(MIN %d) after checking queue capacity.", + conn->ConnID, + conn->SegMemoryMB, + conn->SegCore, + conn->SegNum, + conn->SegNumMin); /*------------------------------------------------------------------ * The following logic consider the actual resource requirement from @@ -3427,15 +3427,15 @@ int computeQueryQuota(ConnectionTrack conn, char *errorbuf, int errorbufsize) conn->SegNum = conn->MaxSegCountFixed; } - elog(LOG, "ConnID %d. Expect query resource (%d MB, %lf CORE) x %d " - "(MIN %d) after checking query expectation %d (MIN %d).", - conn->ConnID, - conn->SegMemoryMB, - conn->SegCore, - conn->SegNum, - conn->SegNumMin, - conn->MaxSegCountFixed, - conn->MinSegCountFixed); + elog(RMLOG, "ConnID %d. Expect query resource (%d MB, %lf CORE) x %d " + "(MIN %d) after checking query expectation %d (MIN %d).", + conn->ConnID, + conn->SegMemoryMB, + conn->SegCore, + conn->SegNum, + conn->SegNumMin, + conn->MaxSegCountFixed, + conn->MinSegCountFixed); } @@ -5003,9 +5003,9 @@ int rebuildAllResourceQueueTrackDynamicStatusInShadow(List *quehavingshadow, } else { - elog(LOG, "Resource manager passed rebuilding resource queue %s " - "dynamic status in its shadow.", - quetrack->QueueInfo->Name); + elog(RMLOG, "Resource manager passed rebuilding resource queue %s " + "dynamic status in its shadow.", + quetrack->QueueInfo->Name); } res = detectAndDealWithDeadLockInShadow(quetrack, queuechanged); @@ -5020,13 +5020,14 @@ int rebuildAllResourceQueueTrackDynamicStatusInShadow(List *quehavingshadow, } else { - elog(LOG, "Resource manager passed detecting deadlock issues in the " - "shadow of resource queue %s", - quetrack->QueueInfo->Name); + elog(RMLOG, "Resource manager passed detecting deadlock issues in the " + "shadow of resource queue %s", + quetrack->QueueInfo->Name); } } - elog(LOG, "Resource manager finished rebuilding resource queues' dynamic status"); + elog(RMLOG, "Resource manager finished rebuilding resource queues' dynamic " + "status"); return FUNC_RETURN_OK; } @@ -5037,8 +5038,8 @@ int rebuildResourceQueueTrackDynamicStatusInShadow(DynResourceQueueTrack quetra { int res = FUNC_RETURN_OK; - elog(LOG, "Rebuild resource queue %s dynamic status in its shadow.", - quetrack->QueueInfo->Name); + elog(RMLOG, "Rebuild resource queue %s dynamic status in its shadow.", + quetrack->QueueInfo->Name); DynResourceQueueTrack shadowtrack = quetrack->ShadowQueueTrack; /* Get deadlock detector ready in the shadow instance. */ @@ -5139,8 +5140,9 @@ int rebuildResourceQueueTrackDynamicStatusInShadow(DynResourceQueueTrack quetra shadowtrack->DLDetector.InUseTotal.MemoryMB, shadowtrack->DLDetector.LockedTotal.MemoryMB); - elog(LOG, "Finished rebuilding resource queue %s dynamic status in its shadow.", - quetrack->QueueInfo->Name); + elog(RMLOG, "Finished rebuilding resource queue %s dynamic status in its " + "shadow.", + quetrack->QueueInfo->Name); return FUNC_RETURN_OK; } http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/37839128/src/backend/tcop/pquery.c ---------------------------------------------------------------------- diff --git a/src/backend/tcop/pquery.c b/src/backend/tcop/pquery.c index 8ef445a..efa5bce 100644 --- a/src/backend/tcop/pquery.c +++ b/src/backend/tcop/pquery.c @@ -1123,8 +1123,6 @@ CleanupGlobalQueryResources(void) int ret; char errorbuf[1024]; - elog(LOG, "In CleanupGlobalQueryResources()."); - /* Force using new socket connection to return and free. */ bool oldval = rm_enable_connpool; rm_enable_connpool = false;
