svn commit: r1409032 - in /hadoop/common/trunk/hadoop-mapreduce-project: CHANGES.txt hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainer

2012-11-13 Thread tgraves 
Author: tgraves
Date: Tue Nov 13 23:47:57 2012
New Revision: 1409032

URL: http://svn.apache.org/viewvc?rev=1409032view=rev
Log:
MAPREDUCE-4517. Too many INFO messages written out during AM to RM heartbeat 
(Jason Lowe via tgraves)

Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt

hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java

Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1409032r1=1409031r2=1409032view=diff
==
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Tue Nov 13 
23:47:57 2012
@@ -660,6 +660,9 @@ Release 0.23.5 - UNRELEASED
 
 MAPREDUCE-4786. Job End Notification retry interval is 5 milliseconds by
 default (Ravi Prakash via bobby)
+
+MAPREDUCE-4517. Too many INFO messages written out during AM to RM 
heartbeat
+(Jason Lowe via tgraves)
  
 Release 0.23.4 - UNRELEASED
 

Modified: 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java?rev=1409032r1=1409031r2=1409032view=diff
==
--- 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
 (original)
+++ 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
 Tue Nov 13 23:47:57 2012
@@ -67,6 +67,7 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.NodeReport;
 import org.apache.hadoop.yarn.api.records.NodeState;
 import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.util.RackResolver;
 
@@ -145,6 +146,8 @@ public class RMContainerAllocator extend
   BlockingQueueContainerAllocatorEvent eventQueue
 = new LinkedBlockingQueueContainerAllocatorEvent();
 
+  private ScheduleStats scheduleStats = new ScheduleStats();
+
   public RMContainerAllocator(ClientService clientService, AppContext context) 
{
 super(clientService, context);
 this.stopped = new AtomicBoolean(false);
@@ -208,13 +211,10 @@ public class RMContainerAllocator extend
 
   @Override
   protected synchronized void heartbeat() throws Exception {
-LOG.info(Before Scheduling:  + getStat());
+scheduleStats.updateAndLogIfChanged(Before Scheduling: );
 ListContainer allocatedContainers = getResources();
-LOG.info(After Scheduling:  + getStat());
 if (allocatedContainers.size()  0) {
-  LOG.info(Before Assign:  + getStat());
   scheduledRequests.assign(allocatedContainers);
-  LOG.info(After Assign:  + getStat());
 }
 
 int completedMaps = getJob().getCompletedMaps();
@@ -235,6 +235,8 @@ public class RMContainerAllocator extend
   maxReduceRampupLimit, reduceSlowStart);
   recalculateReduceSchedule = false;
 }
+
+scheduleStats.updateAndLogIfChanged(After Scheduling: );
   }
 
   @Override
@@ -245,7 +247,7 @@ public class RMContainerAllocator extend
 }
 eventHandlingThread.interrupt();
 super.stop();
-LOG.info(Final Stats:  + getStat());
+scheduleStats.log(Final Stats: );
   }
 
   public boolean getIsReduceStarted() {
@@ -427,7 +429,9 @@ public class RMContainerAllocator extend
   return;
 }
 
-LOG.info(Recalculating schedule...);
+int headRoom = getAvailableResources() != null ?
+getAvailableResources().getMemory() : 0;
+LOG.info(Recalculating schedule, headroom= + headRoom);
 
 //check for slow start
 if (!getIsReduceStarted()) {//not set yet
@@ -536,24 +540,6 @@ public class RMContainerAllocator extend
 }
   }
   
-  /**
-   * Synchronized to avoid findbugs warnings
-   */
-  private synchronized String getStat() {
-return PendingReduces: + pendingReduces.size() +
- ScheduledMaps: + scheduledRequests.maps.size() +
- ScheduledReduces: + scheduledRequests.reduces.size() +
- AssignedMaps: + assignedRequests.maps.size() + 
- AssignedReduces: + assignedRequests.reduces.size() +
- completedMaps: + getJob().getCompletedMaps() + 
- completedReduces:

svn commit: r1409034 - in /hadoop/common/branches/branch-2/hadoop-mapreduce-project: CHANGES.txt hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm

2012-11-13 Thread tgraves 
Author: tgraves
Date: Tue Nov 13 23:49:04 2012
New Revision: 1409034

URL: http://svn.apache.org/viewvc?rev=1409034view=rev
Log:
merge -r 1409031:1409032  from trunk. FIXES: MAPREDUCE-4517

Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java

Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1409034r1=1409033r2=1409034view=diff
==
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt 
(original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Tue 
Nov 13 23:49:04 2012
@@ -513,6 +513,9 @@ Release 0.23.5 - UNRELEASED
 
 MAPREDUCE-4786. Job End Notification retry interval is 5 milliseconds by
 default (Ravi Prakash via bobby)
+
+MAPREDUCE-4517. Too many INFO messages written out during AM to RM 
heartbeat
+(Jason Lowe via tgraves)
  
 Release 0.23.4 - UNRELEASED
 

Modified: 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java?rev=1409034r1=1409033r2=1409034view=diff
==
--- 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
 (original)
+++ 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
 Tue Nov 13 23:49:04 2012
@@ -67,6 +67,7 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.NodeReport;
 import org.apache.hadoop.yarn.api.records.NodeState;
 import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.util.RackResolver;
 
@@ -145,6 +146,8 @@ public class RMContainerAllocator extend
   BlockingQueueContainerAllocatorEvent eventQueue
 = new LinkedBlockingQueueContainerAllocatorEvent();
 
+  private ScheduleStats scheduleStats = new ScheduleStats();
+
   public RMContainerAllocator(ClientService clientService, AppContext context) 
{
 super(clientService, context);
 this.stopped = new AtomicBoolean(false);
@@ -208,13 +211,10 @@ public class RMContainerAllocator extend
 
   @Override
   protected synchronized void heartbeat() throws Exception {
-LOG.info(Before Scheduling:  + getStat());
+scheduleStats.updateAndLogIfChanged(Before Scheduling: );
 ListContainer allocatedContainers = getResources();
-LOG.info(After Scheduling:  + getStat());
 if (allocatedContainers.size()  0) {
-  LOG.info(Before Assign:  + getStat());
   scheduledRequests.assign(allocatedContainers);
-  LOG.info(After Assign:  + getStat());
 }
 
 int completedMaps = getJob().getCompletedMaps();
@@ -235,6 +235,8 @@ public class RMContainerAllocator extend
   maxReduceRampupLimit, reduceSlowStart);
   recalculateReduceSchedule = false;
 }
+
+scheduleStats.updateAndLogIfChanged(After Scheduling: );
   }
 
   @Override
@@ -245,7 +247,7 @@ public class RMContainerAllocator extend
 }
 eventHandlingThread.interrupt();
 super.stop();
-LOG.info(Final Stats:  + getStat());
+scheduleStats.log(Final Stats: );
   }
 
   public boolean getIsReduceStarted() {
@@ -427,7 +429,9 @@ public class RMContainerAllocator extend
   return;
 }
 
-LOG.info(Recalculating schedule...);
+int headRoom = getAvailableResources() != null ?
+getAvailableResources().getMemory() : 0;
+LOG.info(Recalculating schedule, headroom= + headRoom);
 
 //check for slow start
 if (!getIsReduceStarted()) {//not set yet
@@ -536,24 +540,6 @@ public class RMContainerAllocator extend
 }
   }
   
-  /**
-   * Synchronized to avoid findbugs warnings
-   */
-  private synchronized String getStat() {
-return PendingReduces: + pendingReduces.size() +
- ScheduledMaps: + scheduledRequests.maps.size() +
- ScheduledReduces: + scheduledRequests.reduces.size() +
- AssignedMaps: + assignedRequests.maps.size() + 
- AssignedReduces: + assignedRequests.reduces.size() +
-

svn commit: r1409035 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: CHANGES.txt hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app

2012-11-13 Thread tgraves 
Author: tgraves
Date: Tue Nov 13 23:50:23 2012
New Revision: 1409035

URL: http://svn.apache.org/viewvc?rev=1409035view=rev
Log:
merge -r 1409031:1409032 from trunk. FIXES: MAPREDUCE-4517

Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt

hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java

Modified: 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1409035r1=1409034r2=1409035view=diff
==
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt 
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Tue 
Nov 13 23:50:23 2012
@@ -96,6 +96,9 @@ Release 0.23.5 - UNRELEASED
 
 MAPREDUCE-4786. Job End Notification retry interval is 5 milliseconds by
 default (Ravi Prakash via bobby)
+
+MAPREDUCE-4517. Too many INFO messages written out during AM to RM 
heartbeat
+(Jason Lowe via tgraves)
  
 Release 0.23.4 - UNRELEASED
 

Modified: 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java?rev=1409035r1=1409034r2=1409035view=diff
==
--- 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
 (original)
+++ 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
 Tue Nov 13 23:50:23 2012
@@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.util.RackResolver;
 
@@ -140,6 +141,8 @@ public class RMContainerAllocator extend
   BlockingQueueContainerAllocatorEvent eventQueue
 = new LinkedBlockingQueueContainerAllocatorEvent();
 
+  private ScheduleStats scheduleStats = new ScheduleStats();
+
   public RMContainerAllocator(ClientService clientService, AppContext context) 
{
 super(clientService, context);
 this.stopped = new AtomicBoolean(false);
@@ -203,13 +206,10 @@ public class RMContainerAllocator extend
 
   @Override
   protected synchronized void heartbeat() throws Exception {
-LOG.info(Before Scheduling:  + getStat());
+scheduleStats.updateAndLogIfChanged(Before Scheduling: );
 ListContainer allocatedContainers = getResources();
-LOG.info(After Scheduling:  + getStat());
 if (allocatedContainers.size()  0) {
-  LOG.info(Before Assign:  + getStat());
   scheduledRequests.assign(allocatedContainers);
-  LOG.info(After Assign:  + getStat());
 }
 
 int completedMaps = getJob().getCompletedMaps();
@@ -230,6 +230,8 @@ public class RMContainerAllocator extend
   maxReduceRampupLimit, reduceSlowStart);
   recalculateReduceSchedule = false;
 }
+
+scheduleStats.updateAndLogIfChanged(After Scheduling: );
   }
 
   @Override
@@ -240,7 +242,7 @@ public class RMContainerAllocator extend
 }
 eventHandlingThread.interrupt();
 super.stop();
-LOG.info(Final Stats:  + getStat());
+scheduleStats.log(Final Stats: );
   }
 
   public boolean getIsReduceStarted() {
@@ -422,7 +424,9 @@ public class RMContainerAllocator extend
   return;
 }
 
-LOG.info(Recalculating schedule...);
+int headRoom = getAvailableResources() != null ?
+getAvailableResources().getMemory() : 0;
+LOG.info(Recalculating schedule, headroom= + headRoom);
 
 //check for slow start
 if (!getIsReduceStarted()) {//not set yet
@@ -531,24 +535,6 @@ public class RMContainerAllocator extend
 }
   }
   
-  /**
-   * Synchronized to avoid findbugs warnings
-   */
-  private synchronized String getStat() {
-return PendingReduces: + pendingReduces.size() +
- ScheduledMaps: + scheduledRequests.maps.size() +
- ScheduledReduces: + scheduledRequests.reduces.size() +
- AssignedMaps: + assignedRequests.maps.size() + 
- AssignedReduces: +