Author: vinodkv
Date: Thu Apr 11 22:27:45 2013
New Revision: 1467120
URL: http://svn.apache.org/r1467120
Log:
MAPREUDUCE-5059. Change average merge time on Job overview page to be the time
delta between the end of the shuffle and the start of the reduce. Contributed
by Omkar Vinit Joshi.
Added:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/TestJobInfo.java
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/JobInfo.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEntities.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001-1329348443227-user-Sleep+job-1329348468601-10-1-SUCCEEDED-default.jhist
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1467120&r1=1467119&r2=1467120&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Thu Apr 11
22:27:45 2013
@@ -188,6 +188,10 @@ Release 2.0.5-beta - UNRELEASED
MAPREDUCE-4981. Add WordMean, WordMedian, WordStandardDeviation
to ExamplesDriver. (Plamen Jeliazkov via shv)
+ MAPREUDUCE-5059. Change average merge time on Job overview page to be the
+ time delta between the end of the shuffle and the start of the reduce.
+ (Omkar Vinit Joshi via vinodkv)
+
OPTIMIZATIONS
BUG FIXES
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/JobInfo.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/JobInfo.java?rev=1467120&r1=1467119&r2=1467120&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/JobInfo.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/JobInfo.java
Thu Apr 11 22:27:45 2013
@@ -287,7 +287,7 @@ public class JobInfo {
avgShuffleTime += (attempt.getShuffleFinishTime() - attempt
.getLaunchTime());
avgMergeTime += attempt.getSortFinishTime()
- - attempt.getLaunchTime();
+ - attempt.getShuffleFinishTime();
avgReduceTime += (attempt.getFinishTime() - attempt
.getShuffleFinishTime());
}
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEntities.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEntities.java?rev=1467120&r1=1467119&r2=1467120&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEntities.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEntities.java
Thu Apr 11 22:27:45 2013
@@ -94,11 +94,11 @@ public class TestJobHistoryEntities {
assertEquals(1, completedJob.getAMInfos().size());
assertEquals(10, completedJob.getCompletedMaps());
assertEquals(1, completedJob.getCompletedReduces());
- assertEquals(11, completedJob.getTasks().size());
+ assertEquals(12, completedJob.getTasks().size());
//Verify tasks loaded at this point.
assertEquals(true, completedJob.tasksLoaded.get());
assertEquals(10, completedJob.getTasks(TaskType.MAP).size());
- assertEquals(1, completedJob.getTasks(TaskType.REDUCE).size());
+ assertEquals(2, completedJob.getTasks(TaskType.REDUCE).size());
assertEquals("user", completedJob.getUserName());
assertEquals(JobState.SUCCEEDED, completedJob.getState());
JobReport jobReport = completedJob.getReport();
@@ -119,7 +119,7 @@ public class TestJobHistoryEntities {
Map<TaskId, Task> mapTasks = completedJob.getTasks(TaskType.MAP);
Map<TaskId, Task> reduceTasks = completedJob.getTasks(TaskType.REDUCE);
assertEquals(10, mapTasks.size());
- assertEquals(1, reduceTasks.size());
+ assertEquals(2, reduceTasks.size());
Task mt1 = mapTasks.get(mt1Id);
assertEquals(1, mt1.getAttempts().size());
@@ -134,7 +134,7 @@ public class TestJobHistoryEntities {
assertEquals(TaskState.SUCCEEDED, rt1Report.getTaskState());
assertEquals(rt1Id, rt1Report.getTaskId());
}
-
+
@Test (timeout=10000)
public void testCompletedTaskAttempt() throws Exception {
HistoryFileInfo info = mock(HistoryFileInfo.class);
@@ -196,12 +196,12 @@ public class TestJobHistoryEntities {
assertEquals("default",completedJob.getQueueName());
// progress
assertEquals(1.0, completedJob.getProgress(),0.001);
- // 11 rows in answer
-
assertEquals(11,completedJob.getTaskAttemptCompletionEvents(0,1000).length);
+ // 12 rows in answer
+
assertEquals(12,completedJob.getTaskAttemptCompletionEvents(0,1000).length);
// select first 10 rows
assertEquals(10,completedJob.getTaskAttemptCompletionEvents(0,10).length);
// select 5-10 rows include 5th
- assertEquals(6,completedJob.getTaskAttemptCompletionEvents(5,10).length);
+ assertEquals(7,completedJob.getTaskAttemptCompletionEvents(5,10).length);
// without errors
assertEquals(1,completedJob.getDiagnostics().size());
Added:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/TestJobInfo.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/TestJobInfo.java?rev=1467120&view=auto
==============================================================================
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/TestJobInfo.java
(added)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/TestJobInfo.java
Thu Apr 11 22:27:45 2013
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.v2.hs.webapp.dao;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+
+import junit.framework.Assert;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.JobACLsManager;
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.HistoryFileInfo;
+import org.apache.hadoop.mapreduce.v2.hs.CompletedJob;
+import org.apache.hadoop.mapreduce.v2.hs.TestJobHistoryEntities;
+import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils;
+import org.junit.Test;
+
+public class TestJobInfo {
+
+ @Test(timeout = 10000)
+ public void testAverageMergeTime() throws IOException {
+ String historyFileName =
+
"job_1329348432655_0001-1329348443227-user-Sleep+job-1329348468601-10-1-SUCCEEDED-default.jhist";
+ String confFileName =
+ "job_1329348432655_0001_conf.xml";
+ Configuration conf = new Configuration();
+ JobACLsManager jobAclsMgr = new JobACLsManager(conf);
+ Path fulleHistoryPath =
+ new Path(TestJobHistoryEntities.class.getClassLoader()
+ .getResource(historyFileName)
+ .getFile());
+ Path fullConfPath =
+ new Path(TestJobHistoryEntities.class.getClassLoader()
+ .getResource(confFileName)
+ .getFile());
+
+ HistoryFileInfo info = mock(HistoryFileInfo.class);
+ when(info.getConfFile()).thenReturn(fullConfPath);
+
+ JobId jobId = MRBuilderUtils.newJobId(1329348432655l, 1, 1);
+ CompletedJob completedJob =
+ new CompletedJob(conf, jobId, fulleHistoryPath, true, "user",
+ info, jobAclsMgr);
+ JobInfo jobInfo = new JobInfo(completedJob);
+ // There are 2 tasks with merge time of 45 and 55 respectively. So average
+ // merge time should be 50.
+ Assert.assertEquals(50L, jobInfo.getAvgMergeTime().longValue());
+ }
+}
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001-1329348443227-user-Sleep+job-1329348468601-10-1-SUCCEEDED-default.jhist
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001-1329348443227-user-Sleep%2Bjob-1329348468601-10-1-SUCCEEDED-default.jhist?rev=1467120&r1=1467119&r2=1467120&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001-1329348443227-user-Sleep+job-1329348468601-10-1-SUCCEEDED-default.jhist
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001-1329348443227-user-Sleep+job-1329348468601-10-1-SUCCEEDED-default.jhist
Thu Apr 11 22:27:45 2013
@@ -15,6 +15,7 @@ Avro-Json
{"type":"TASK_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskStarted":{"taskid":"task_1329348432655_0001_m_000008","taskType":"MAP","startTime":1329348448388,"splitLocations":""}}}
{"type":"TASK_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskStarted":{"taskid":"task_1329348432655_0001_m_000009","taskType":"MAP","startTime":1329348448388,"splitLocations":""}}}
{"type":"TASK_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskStarted":{"taskid":"task_1329348432655_0001_r_000000","taskType":"REDUCE","startTime":1329348448388,"splitLocations":""}}}
+
{"type":"TASK_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskStarted":{"taskid":"task_1329348432655_0001_r_000001","taskType":"REDUCE","startTime":1329348448388,"splitLocations":""}}}
{"type":"MAP_ATTEMPT_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskAttemptStarted":{"taskid":"task_1329348432655_0001_m_000000","taskType":"MAP","attemptId":"attempt_1329348432655_0001_m_000000_0","startTime":1329348450485,"trackerName":"localhost","httpPort":9999,"shufflePort":8080,"containerId":"container_1329348432655_0001_01_000002"}}}
{"type":"MAP_ATTEMPT_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskAttemptStarted":{"taskid":"task_1329348432655_0001_m_000002","taskType":"MAP","attemptId":"attempt_1329348432655_0001_m_000002_0","startTime":1329348450537,"trackerName":"localhost","httpPort":9999,"shufflePort":8080,"containerId":"container_1329348432655_0001_01_000004"}}}
{"type":"MAP_ATTEMPT_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskAttemptStarted":{"taskid":"task_1329348432655_0001_m_000004","taskType":"MAP","attemptId":"attempt_1329348432655_0001_m_000004_0","startTime":1329348450538,"trackerName":"localhost","httpPort":9999,"shufflePort":8080,"containerId":"container_1329348432655_0001_01_000006"}}}
@@ -40,6 +41,7 @@ Avro-Json
{"type":"MAP_ATTEMPT_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskAttemptStarted":{"taskid":"task_1329348432655_0001_m_000008","taskType":"MAP","attemptId":"attempt_1329348432655_0001_m_000008_0","startTime":1329348462765,"trackerName":"localhost","httpPort":9999,"shufflePort":8080,"containerId":"container_1329348432655_0001_01_000010"}}}
{"type":"MAP_ATTEMPT_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskAttemptStarted":{"taskid":"task_1329348432655_0001_m_000009","taskType":"MAP","attemptId":"attempt_1329348432655_0001_m_000009_0","startTime":1329348462792,"trackerName":"localhost","httpPort":9999,"shufflePort":8080,"containerId":"container_1329348432655_0001_01_000011"}}}
{"type":"REDUCE_ATTEMPT_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskAttemptStarted":{"taskid":"task_1329348432655_0001_r_000000","taskType":"REDUCE","attemptId":"attempt_1329348432655_0001_r_000000_0","startTime":1329348464995,"trackerName":"localhost","httpPort":9999,"shufflePort":8080,"containerId":"container_1329348432655_0001_01_000014"}}}
+
{"type":"REDUCE_ATTEMPT_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskAttemptStarted":{"taskid":"task_1329348432655_0001_r_000001","taskType":"REDUCE","attemptId":"attempt_1329348432655_0001_r_000001_0","startTime":1329348464995,"trackerName":"localhost","httpPort":9999,"shufflePort":8080,"containerId":"container_1329348432655_0001_01_000014"}}}
{"type":"MAP_ATTEMPT_FINISHED","event":{"org.apache.hadoop.mapreduce.jobhistory.MapAttemptFinished":{"taskid":"task_1329348432655_0001_m_000007","attemptId":"attempt_1329348432655_0001_m_000007_0","taskType":"MAP","taskStatus":"SUCCEEDED","mapFinishTime":1329348465534,"finishTime":1329348465965,"hostname":"localhost","port":45454,"rackname":"/default-rack","state":"Sleeping...
(1) ms
left","counters":{"name":"COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File
System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE:
Number of bytes
read","value":120},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of
bytes written","value":48051},{"name":"FILE_READ_OPS","displayName":"FILE:
Number of read
operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE:
Number of large read
operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of
write operations","value":0},{"name":"HDFS_BYT
ES_READ","displayName":"HDFS: Number of bytes
read","value":48},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of
bytes written","value":0},{"name":"HDFS_READ_OPS","displayName":"HDFS: Number
of read
operations","value":1},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS:
Number of large read
operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of
write
operations","value":0}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce
Framework","counts":[{"name":"MAP_INPUT_RECORDS","displayName":"Map input
records","value":1},{"name":"MAP_OUTPUT_RECORDS","displayName":"Map output
records","value":1},{"name":"MAP_OUTPUT_BYTES","displayName":"Map output
bytes","value":4},{"name":"MAP_OUTPUT_MATERIALIZED_BYTES","displayName":"Map
output materialized
bytes","value":12},{"name":"SPLIT_RAW_BYTES","displayName":"Input split
bytes","value":48},{"name":"COMBINE_INPUT_RECORDS","displayName":"Combine input
records","value":0},{"name"
:"SPILLED_RECORDS","displayName":"Spilled
Records","value":1},{"name":"FAILED_SHUFFLE","displayName":"Failed
Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map
outputs","value":0},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed
(ms)","value":194},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent
(ms)","value":320},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical
memory (bytes)
snapshot","value":185327616},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual
memory (bytes)
snapshot","value":713089024},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total
committed heap usage
(bytes)","value":165478400}]},{"name":"org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter","displayName":"File
Input Format Counters ","counts":[{"name":"BYTES_READ","displayName":"Bytes
Read","value":0}]}]},"clockSplits":[3464,18,19,18,18,19,18,18,19,18,18,19],"cpuUsages":[26,27,27,26,27,27,26,27,27,26,27,27],"vMemKbytes":[29015,87046,145078,203109,
261140,319171,377203,435234,493266,551297,609328,667360],"physMemKbytes":[7541,22623,37705,52786,67869,82950,98033,113114,128197,143279,158360,173443]}}}
{"type":"TASK_FINISHED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskFinished":{"taskid":"task_1329348432655_0001_m_000007","taskType":"MAP","finishTime":1329348465965,"status":"SUCCEEDED","counters":{"name":"COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File
System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE:
Number of bytes
read","value":120},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of
bytes written","value":48051},{"name":"FILE_READ_OPS","displayName":"FILE:
Number of read
operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE:
Number of large read
operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of
write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS:
Number of bytes
read","value":48},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of
bytes written","value":0},{"name":"HDFS_READ_OPS","displayName":"HDFS: N
umber of read
operations","value":1},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS:
Number of large read
operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of
write
operations","value":0}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce
Framework","counts":[{"name":"MAP_INPUT_RECORDS","displayName":"Map input
records","value":1},{"name":"MAP_OUTPUT_RECORDS","displayName":"Map output
records","value":1},{"name":"MAP_OUTPUT_BYTES","displayName":"Map output
bytes","value":4},{"name":"MAP_OUTPUT_MATERIALIZED_BYTES","displayName":"Map
output materialized
bytes","value":12},{"name":"SPLIT_RAW_BYTES","displayName":"Input split
bytes","value":48},{"name":"COMBINE_INPUT_RECORDS","displayName":"Combine input
records","value":0},{"name":"SPILLED_RECORDS","displayName":"Spilled
Records","value":1},{"name":"FAILED_SHUFFLE","displayName":"Failed
Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map
outputs","v
alue":0},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed
(ms)","value":194},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent
(ms)","value":320},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical
memory (bytes)
snapshot","value":185327616},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual
memory (bytes)
snapshot","value":713089024},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total
committed heap usage
(bytes)","value":165478400}]},{"name":"org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter","displayName":"File
Input Format Counters ","counts":[{"name":"BYTES_READ","displayName":"Bytes
Read","value":0}]}]}}}}
{"type":"MAP_ATTEMPT_FINISHED","event":{"org.apache.hadoop.mapreduce.jobhistory.MapAttemptFinished":{"taskid":"task_1329348432655_0001_m_000009","attemptId":"attempt_1329348432655_0001_m_000009_0","taskType":"MAP","taskStatus":"SUCCEEDED","mapFinishTime":1329348465986,"finishTime":1329348466363,"hostname":"localhost","port":45454,"rackname":"/default-rack","state":"Sleeping...
(1) ms
left","counters":{"name":"COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File
System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE:
Number of bytes
read","value":120},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of
bytes written","value":48051},{"name":"FILE_READ_OPS","displayName":"FILE:
Number of read
operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE:
Number of large read
operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of
write operations","value":0},{"name":"HDFS_BYT
ES_READ","displayName":"HDFS: Number of bytes
read","value":48},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of
bytes written","value":0},{"name":"HDFS_READ_OPS","displayName":"HDFS: Number
of read
operations","value":1},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS:
Number of large read
operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of
write
operations","value":0}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce
Framework","counts":[{"name":"MAP_INPUT_RECORDS","displayName":"Map input
records","value":1},{"name":"MAP_OUTPUT_RECORDS","displayName":"Map output
records","value":1},{"name":"MAP_OUTPUT_BYTES","displayName":"Map output
bytes","value":4},{"name":"MAP_OUTPUT_MATERIALIZED_BYTES","displayName":"Map
output materialized
bytes","value":12},{"name":"SPLIT_RAW_BYTES","displayName":"Input split
bytes","value":48},{"name":"COMBINE_INPUT_RECORDS","displayName":"Combine input
records","value":0},{"name"
:"SPILLED_RECORDS","displayName":"Spilled
Records","value":1},{"name":"FAILED_SHUFFLE","displayName":"Failed
Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map
outputs","value":0},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed
(ms)","value":23},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent
(ms)","value":330},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical
memory (bytes)
snapshot","value":182169600},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual
memory (bytes)
snapshot","value":705945600},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total
committed heap usage
(bytes)","value":165478400}]},{"name":"org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter","displayName":"File
Input Format Counters ","counts":[{"name":"BYTES_READ","displayName":"Bytes
Read","value":0}]}]},"clockSplits":[3223,21,21,21,21,21,20,21,21,21,21,21],"cpuUsages":[27,28,27,28,27,28,27,28,27,28,27,28],"vMemKbytes":[28725,86175,143625,201074,2
58525,315974,373425,430874,488325,545775,603224,660675],"physMemKbytes":[7412,22237,37062,51887,66712,81537,96362,111187,126012,140837,155662,170487]}}}
@@ -47,5 +49,7 @@ Avro-Json
{"type":"MAP_ATTEMPT_FINISHED","event":{"org.apache.hadoop.mapreduce.jobhistory.MapAttemptFinished":{"taskid":"task_1329348432655_0001_m_000008","attemptId":"attempt_1329348432655_0001_m_000008_0","taskType":"MAP","taskStatus":"SUCCEEDED","mapFinishTime":1329348467231,"finishTime":1329348467421,"hostname":"localhost","port":45454,"rackname":"/default-rack","state":"Sleeping...
(1) ms
left","counters":{"name":"COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File
System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE:
Number of bytes
read","value":120},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of
bytes written","value":48051},{"name":"FILE_READ_OPS","displayName":"FILE:
Number of read
operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE:
Number of large read
operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of
write operations","value":0},{"name":"HDFS_BYT
ES_READ","displayName":"HDFS: Number of bytes
read","value":48},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of
bytes written","value":0},{"name":"HDFS_READ_OPS","displayName":"HDFS: Number
of read
operations","value":1},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS:
Number of large read
operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of
write
operations","value":0}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce
Framework","counts":[{"name":"MAP_INPUT_RECORDS","displayName":"Map input
records","value":1},{"name":"MAP_OUTPUT_RECORDS","displayName":"Map output
records","value":1},{"name":"MAP_OUTPUT_BYTES","displayName":"Map output
bytes","value":4},{"name":"MAP_OUTPUT_MATERIALIZED_BYTES","displayName":"Map
output materialized
bytes","value":12},{"name":"SPLIT_RAW_BYTES","displayName":"Input split
bytes","value":48},{"name":"COMBINE_INPUT_RECORDS","displayName":"Combine input
records","value":0},{"name"
:"SPILLED_RECORDS","displayName":"Spilled
Records","value":1},{"name":"FAILED_SHUFFLE","displayName":"Failed
Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map
outputs","value":0},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed
(ms)","value":12},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent
(ms)","value":320},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical
memory (bytes)
snapshot","value":181297152},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual
memory (bytes)
snapshot","value":705019904},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total
committed heap usage
(bytes)","value":165478400}]},{"name":"org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter","displayName":"File
Input Format Counters ","counts":[{"name":"BYTES_READ","displayName":"Bytes
Read","value":0}]}]},"clockSplits":[4483,15,16,15,16,15,15,16,15,16,15,16],"cpuUsages":[26,27,27,26,27,27,26,27,27,26,27,27],"vMemKbytes":[28686,86061,143436,200810,2
58185,315560,372935,430309,487684,545059,602433,659808],"physMemKbytes":[7377,22131,36885,51638,66393,81146,95901,110654,125409,140163,154916,169671]}}}
{"type":"TASK_FINISHED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskFinished":{"taskid":"task_1329348432655_0001_m_000008","taskType":"MAP","finishTime":1329348467421,"status":"SUCCEEDED","counters":{"name":"COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File
System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE:
Number of bytes
read","value":120},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of
bytes written","value":48051},{"name":"FILE_READ_OPS","displayName":"FILE:
Number of read
operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE:
Number of large read
operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of
write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS:
Number of bytes
read","value":48},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of
bytes written","value":0},{"name":"HDFS_READ_OPS","displayName":"HDFS: N
umber of read
operations","value":1},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS:
Number of large read
operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of
write
operations","value":0}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce
Framework","counts":[{"name":"MAP_INPUT_RECORDS","displayName":"Map input
records","value":1},{"name":"MAP_OUTPUT_RECORDS","displayName":"Map output
records","value":1},{"name":"MAP_OUTPUT_BYTES","displayName":"Map output
bytes","value":4},{"name":"MAP_OUTPUT_MATERIALIZED_BYTES","displayName":"Map
output materialized
bytes","value":12},{"name":"SPLIT_RAW_BYTES","displayName":"Input split
bytes","value":48},{"name":"COMBINE_INPUT_RECORDS","displayName":"Combine input
records","value":0},{"name":"SPILLED_RECORDS","displayName":"Spilled
Records","value":1},{"name":"FAILED_SHUFFLE","displayName":"Failed
Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map
outputs","v
alue":0},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed
(ms)","value":12},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent
(ms)","value":320},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical
memory (bytes)
snapshot","value":181297152},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual
memory (bytes)
snapshot","value":705019904},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total
committed heap usage
(bytes)","value":165478400}]},{"name":"org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter","displayName":"File
Input Format Counters ","counts":[{"name":"BYTES_READ","displayName":"Bytes
Read","value":0}]}]}}}}
{"type":"REDUCE_ATTEMPT_FINISHED","event":{"org.apache.hadoop.mapreduce.jobhistory.ReduceAttemptFinished":{"taskid":"task_1329348432655_0001_r_000000","attemptId":"attempt_1329348432655_0001_r_000000_0","taskType":"REDUCE","taskStatus":"SUCCEEDED","shuffleFinishTime":1329348468462,"sortFinishTime":1329348468517,"finishTime":1329348468600,"hostname":"localhost","port":45454,"rackname":"/default-rack","state":"Sleeping...
(1) ms left >
reduce","counters":{"name":"COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File
System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE:
Number of bytes
read","value":186},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of
bytes written","value":48074},{"name":"FILE_READ_OPS","displayName":"FILE:
Number of read
operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE:
Number of large read
operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Num
ber of write
operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS: Number of
bytes read","value":0},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number
of bytes written","value":0},{"name":"HDFS_READ_OPS","displayName":"HDFS:
Number of read
operations","value":0},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS:
Number of large read
operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of
write
operations","value":0}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce
Framework","counts":[{"name":"COMBINE_INPUT_RECORDS","displayName":"Combine
input
records","value":0},{"name":"COMBINE_OUTPUT_RECORDS","displayName":"Combine
output records","value":0},{"name":"REDUCE_INPUT_GROUPS","displayName":"Reduce
input groups","value":1},{"name":"REDUCE_SHUFFLE_BYTES","displayName":"Reduce
shuffle
bytes","value":120},{"name":"REDUCE_INPUT_RECORDS","displayName":"Reduce input
records","value":10},{"name":"REDUCE_OUTPUT_
RECORDS","displayName":"Reduce output
records","value":0},{"name":"SPILLED_RECORDS","displayName":"Spilled
Records","value":10},{"name":"SHUFFLED_MAPS","displayName":"Shuffled Maps
","value":10},{"name":"FAILED_SHUFFLE","displayName":"Failed
Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map
outputs","value":10},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed
(ms)","value":14},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent
(ms)","value":1070},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical
memory (bytes)
snapshot","value":82780160},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual
memory (bytes)
snapshot","value":714436608},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total
committed heap usage (bytes)","value":60555264}]},{"name":"Shuffle
Errors","displayName":"Shuffle
Errors","counts":[{"name":"BAD_ID","displayName":"BAD_ID","value":0},{"name":"CONNECTION","displayName":"CONNECTION","value":0},{"name":"IO_ERROR","disp
layName":"IO_ERROR","value":0},{"name":"WRONG_LENGTH","displayName":"WRONG_LENGTH","value":0},{"name":"WRONG_MAP","displayName":"WRONG_MAP","value":0},{"name":"WRONG_REDUCE","displayName":"WRONG_REDUCE","value":0}]},{"name":"org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter","displayName":"File
Output Format Counters
","counts":[{"name":"BYTES_WRITTEN","displayName":"Bytes
Written","value":0}]}]},"clockSplits":[3530,6,7,6,7,6,6,7,6,7,6,7],"cpuUsages":[89,89,89,89,89,90,89,89,89,89,89,90],"vMemKbytes":[29070,87211,145352,203493,261634,319775,377916,436057,494198,552339,610480,668621],"physMemKbytes":[3367,10104,16841,23577,30314,37051,43788,50524,57261,63998,70734,77471]}}}
+
{"type":"REDUCE_ATTEMPT_FINISHED","event":{"org.apache.hadoop.mapreduce.jobhistory.ReduceAttemptFinished":{"taskid":"task_1329348432655_0001_r_000001","attemptId":"attempt_1329348432655_0001_r_000001_0","taskType":"REDUCE","taskStatus":"SUCCEEDED","shuffleFinishTime":1329348468462,"sortFinishTime":1329348468507,"finishTime":1329348468600,"hostname":"localhost","port":45454,"rackname":"/default-rack","state":"Sleeping...
(1) ms left >
reduce","counters":{"name":"COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File
System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE:
Number of bytes
read","value":186},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of
bytes written","value":48074},{"name":"FILE_READ_OPS","displayName":"FILE:
Number of read
operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE:
Number of large read
operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Num
ber of write
operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS: Number of
bytes read","value":0},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number
of bytes written","value":0},{"name":"HDFS_READ_OPS","displayName":"HDFS:
Number of read
operations","value":0},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS:
Number of large read
operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of
write
operations","value":0}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce
Framework","counts":[{"name":"COMBINE_INPUT_RECORDS","displayName":"Combine
input
records","value":0},{"name":"COMBINE_OUTPUT_RECORDS","displayName":"Combine
output records","value":0},{"name":"REDUCE_INPUT_GROUPS","displayName":"Reduce
input groups","value":1},{"name":"REDUCE_SHUFFLE_BYTES","displayName":"Reduce
shuffle
bytes","value":120},{"name":"REDUCE_INPUT_RECORDS","displayName":"Reduce input
records","value":10},{"name":"REDUCE_OUTPUT_
RECORDS","displayName":"Reduce output
records","value":0},{"name":"SPILLED_RECORDS","displayName":"Spilled
Records","value":10},{"name":"SHUFFLED_MAPS","displayName":"Shuffled Maps
","value":10},{"name":"FAILED_SHUFFLE","displayName":"Failed
Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map
outputs","value":10},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed
(ms)","value":14},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent
(ms)","value":1070},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical
memory (bytes)
snapshot","value":82780160},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual
memory (bytes)
snapshot","value":714436608},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total
committed heap usage (bytes)","value":60555264}]},{"name":"Shuffle
Errors","displayName":"Shuffle
Errors","counts":[{"name":"BAD_ID","displayName":"BAD_ID","value":0},{"name":"CONNECTION","displayName":"CONNECTION","value":0},{"name":"IO_ERROR","disp
layName":"IO_ERROR","value":0},{"name":"WRONG_LENGTH","displayName":"WRONG_LENGTH","value":0},{"name":"WRONG_MAP","displayName":"WRONG_MAP","value":0},{"name":"WRONG_REDUCE","displayName":"WRONG_REDUCE","value":0}]},{"name":"org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter","displayName":"File
Output Format Counters
","counts":[{"name":"BYTES_WRITTEN","displayName":"Bytes
Written","value":0}]}]},"clockSplits":[3530,6,7,6,7,6,6,7,6,7,6,7],"cpuUsages":[89,89,89,89,89,90,89,89,89,89,89,90],"vMemKbytes":[29070,87211,145352,203493,261634,319775,377916,436057,494198,552339,610480,668621],"physMemKbytes":[3367,10104,16841,23577,30314,37051,43788,50524,57261,63998,70734,77471]}}}
{"type":"TASK_FINISHED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskFinished":{"taskid":"task_1329348432655_0001_r_000000","taskType":"REDUCE","finishTime":1329348468600,"status":"SUCCEEDED","counters":{"name":"COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File
System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE:
Number of bytes
read","value":186},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of
bytes written","value":48074},{"name":"FILE_READ_OPS","displayName":"FILE:
Number of read
operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE:
Number of large read
operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of
write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS:
Number of bytes
read","value":0},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of
bytes written","value":0},{"name":"HDFS_READ_OPS","displayName":"HDFS:
Number of read
operations","value":0},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS:
Number of large read
operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of
write
operations","value":0}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce
Framework","counts":[{"name":"COMBINE_INPUT_RECORDS","displayName":"Combine
input
records","value":0},{"name":"COMBINE_OUTPUT_RECORDS","displayName":"Combine
output records","value":0},{"name":"REDUCE_INPUT_GROUPS","displayName":"Reduce
input groups","value":1},{"name":"REDUCE_SHUFFLE_BYTES","displayName":"Reduce
shuffle
bytes","value":120},{"name":"REDUCE_INPUT_RECORDS","displayName":"Reduce input
records","value":10},{"name":"REDUCE_OUTPUT_RECORDS","displayName":"Reduce
output records","value":0},{"name":"SPILLED_RECORDS","displayName":"Spilled
Records","value":10},{"name":"SHUFFLED_MAPS","displayName":"Shuffled Maps
","value":10},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuf
fles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map
outputs","value":10},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed
(ms)","value":14},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent
(ms)","value":1070},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical
memory (bytes)
snapshot","value":82780160},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual
memory (bytes)
snapshot","value":714436608},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total
committed heap usage (bytes)","value":60555264}]},{"name":"Shuffle
Errors","displayName":"Shuffle
Errors","counts":[{"name":"BAD_ID","displayName":"BAD_ID","value":0},{"name":"CONNECTION","displayName":"CONNECTION","value":0},{"name":"IO_ERROR","displayName":"IO_ERROR","value":0},{"name":"WRONG_LENGTH","displayName":"WRONG_LENGTH","value":0},{"name":"WRONG_MAP","displayName":"WRONG_MAP","value":0},{"name":"WRONG_REDUCE","displayName":"WRONG_REDUCE","value":0}]},{"name":"org.apache.hadoop.map
reduce.lib.output.FileOutputFormatCounter","displayName":"File Output Format
Counters ","counts":[{"name":"BYTES_WRITTEN","displayName":"Bytes
Written","value":0}]}]}}}}
+
{"type":"TASK_FINISHED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskFinished":{"taskid":"task_1329348432655_0001_r_000001","taskType":"REDUCE","finishTime":1329348468600,"status":"SUCCEEDED","counters":{"name":"COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File
System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE:
Number of bytes
read","value":186},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of
bytes written","value":48074},{"name":"FILE_READ_OPS","displayName":"FILE:
Number of read
operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE:
Number of large read
operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of
write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS:
Number of bytes
read","value":0},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of
bytes written","value":0},{"name":"HDFS_READ_OPS","displayName":"HDFS:
Number of read
operations","value":0},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS:
Number of large read
operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of
write
operations","value":0}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce
Framework","counts":[{"name":"COMBINE_INPUT_RECORDS","displayName":"Combine
input
records","value":0},{"name":"COMBINE_OUTPUT_RECORDS","displayName":"Combine
output records","value":0},{"name":"REDUCE_INPUT_GROUPS","displayName":"Reduce
input groups","value":1},{"name":"REDUCE_SHUFFLE_BYTES","displayName":"Reduce
shuffle
bytes","value":120},{"name":"REDUCE_INPUT_RECORDS","displayName":"Reduce input
records","value":10},{"name":"REDUCE_OUTPUT_RECORDS","displayName":"Reduce
output records","value":0},{"name":"SPILLED_RECORDS","displayName":"Spilled
Records","value":10},{"name":"SHUFFLED_MAPS","displayName":"Shuffled Maps
","value":10},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuf
fles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map
outputs","value":10},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed
(ms)","value":14},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent
(ms)","value":1070},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical
memory (bytes)
snapshot","value":82780160},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual
memory (bytes)
snapshot","value":714436608},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total
committed heap usage (bytes)","value":60555264}]},{"name":"Shuffle
Errors","displayName":"Shuffle
Errors","counts":[{"name":"BAD_ID","displayName":"BAD_ID","value":0},{"name":"CONNECTION","displayName":"CONNECTION","value":0},{"name":"IO_ERROR","displayName":"IO_ERROR","value":0},{"name":"WRONG_LENGTH","displayName":"WRONG_LENGTH","value":0},{"name":"WRONG_MAP","displayName":"WRONG_MAP","value":0},{"name":"WRONG_REDUCE","displayName":"WRONG_REDUCE","value":0}]},{"name":"org.apache.hadoop.map
reduce.lib.output.FileOutputFormatCounter","displayName":"File Output Format
Counters ","counts":[{"name":"BYTES_WRITTEN","displayName":"Bytes
Written","value":0}]}]}}}}
{"type":"JOB_FINISHED","event":{"org.apache.hadoop.mapreduce.jobhistory.JobFinished":{"jobid":"job_1329348432655_0001","finishTime":1329348468601,"finishedMaps":10,"finishedReduces":1,"failedMaps":0,"failedReduces":0,"totalCounters":{"name":"TOTAL_COUNTERS","groups":[{"name":"Shuffle
Errors","displayName":"Shuffle
Errors","counts":[{"name":"BAD_ID","displayName":"BAD_ID","value":0},{"name":"CONNECTION","displayName":"CONNECTION","value":0},{"name":"IO_ERROR","displayName":"IO_ERROR","value":0},{"name":"WRONG_LENGTH","displayName":"WRONG_LENGTH","value":0},{"name":"WRONG_MAP","displayName":"WRONG_MAP","value":0},{"name":"WRONG_REDUCE","displayName":"WRONG_REDUCE","value":0}]},{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File
System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE:
Number of bytes
read","value":1386},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of
bytes written","value":528584},{"name":"FILE_READ_OPS","di
splayName":"FILE: Number of read
operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE:
Number of large read
operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of
write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS:
Number of bytes
read","value":480},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of
bytes written","value":0},{"name":"HDFS_READ_OPS","displayName":"HDFS: Number
of read
operations","value":10},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS:
Number of large read
operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of
write
operations","value":0}]},{"name":"org.apache.hadoop.mapreduce.JobCounter","displayName":"Job
Counters ","counts":[{"name":"TOTAL_LAUNCHED_MAPS","displayName":"Launched map
tasks","value":10},{"name":"TOTAL_LAUNCHED_REDUCES","displayName":"Launched
reduce tasks","value":1},{"name":"OTHER_LOCAL_MAPS","displayName":"Other local
map tasks","value
":10},{"name":"SLOTS_MILLIS_MAPS","displayName":"Total time spent by all maps
in occupied slots
(ms)","value":0},{"name":"SLOTS_MILLIS_REDUCES","displayName":"Total time spent
by all reduces in occupied slots
(ms)","value":0}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce
Framework","counts":[{"name":"MAP_INPUT_RECORDS","displayName":"Map input
records","value":10},{"name":"MAP_OUTPUT_RECORDS","displayName":"Map output
records","value":10},{"name":"MAP_OUTPUT_BYTES","displayName":"Map output
bytes","value":40},{"name":"MAP_OUTPUT_MATERIALIZED_BYTES","displayName":"Map
output materialized
bytes","value":120},{"name":"SPLIT_RAW_BYTES","displayName":"Input split
bytes","value":480},{"name":"COMBINE_INPUT_RECORDS","displayName":"Combine
input
records","value":0},{"name":"COMBINE_OUTPUT_RECORDS","displayName":"Combine
output records","value":0},{"name":"REDUCE_INPUT_GROUPS","displayName":"Reduce
input groups","value":1},{"name":"REDUCE_SHUFFLE_BYTES
","displayName":"Reduce shuffle
bytes","value":120},{"name":"REDUCE_INPUT_RECORDS","displayName":"Reduce input
records","value":10},{"name":"REDUCE_OUTPUT_RECORDS","displayName":"Reduce
output records","value":0},{"name":"SPILLED_RECORDS","displayName":"Spilled
Records","value":20},{"name":"SHUFFLED_MAPS","displayName":"Shuffled Maps
","value":10},{"name":"FAILED_SHUFFLE","displayName":"Failed
Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map
outputs","value":10},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed
(ms)","value":2256},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent
(ms)","value":4460},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical
memory (bytes)
snapshot","value":1923493888},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual
memory (bytes)
snapshot","value":7773462528},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total
committed heap usage
(bytes)","value":1778515968}]},{"name":"org.apache.hadoop.mapreduce.l
ib.input.FileInputFormatCounter","displayName":"File Input Format Counters
","counts":[{"name":"BYTES_READ","displayName":"Bytes
Read","value":0}]},{"name":"org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter","displayName":"File
Output Format Counters
","counts":[{"name":"BYTES_WRITTEN","displayName":"Bytes
Written","value":0}]}]},"mapCounters":{"name":"MAP_COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File
System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE:
Number of bytes
read","value":1200},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of
bytes written","value":480510},{"name":"FILE_READ_OPS","displayName":"FILE:
Number of read
operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE:
Number of large read
operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of
write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS:
Number of
bytes read","value":480},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS:
Number of bytes
written","value":0},{"name":"HDFS_READ_OPS","displayName":"HDFS: Number of read
operations","value":10},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS:
Number of large read
operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of
write
operations","value":0}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce
Framework","counts":[{"name":"MAP_INPUT_RECORDS","displayName":"Map input
records","value":10},{"name":"MAP_OUTPUT_RECORDS","displayName":"Map output
records","value":10},{"name":"MAP_OUTPUT_BYTES","displayName":"Map output
bytes","value":40},{"name":"MAP_OUTPUT_MATERIALIZED_BYTES","displayName":"Map
output materialized
bytes","value":120},{"name":"SPLIT_RAW_BYTES","displayName":"Input split
bytes","value":480},{"name":"COMBINE_INPUT_RECORDS","displayName":"Combine
input records","value":0},{"name":"SPILLED_RECORDS","displayName":
"Spilled Records","value":10},{"name":"FAILED_SHUFFLE","displayName":"Failed
Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map
outputs","value":0},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed
(ms)","value":2242},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent
(ms)","value":3390},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical
memory (bytes)
snapshot","value":1840713728},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual
memory (bytes)
snapshot","value":7059025920},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total
committed heap usage
(bytes)","value":1717960704}]},{"name":"org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter","displayName":"File
Input Format Counters ","counts":[{"name":"BYTES_READ","displayName":"Bytes
Read","value":0}]}]},"reduceCounters":{"name":"REDUCE_COUNTERS","groups":[{"name":"Shuffle
Errors","displayName":"Shuffle
Errors","counts":[{"name":"BAD_ID","displayName":"BAD_ID","value":0},{
"name":"CONNECTION","displayName":"CONNECTION","value":0},{"name":"IO_ERROR","displayName":"IO_ERROR","value":0},{"name":"WRONG_LENGTH","displayName":"WRONG_LENGTH","value":0},{"name":"WRONG_MAP","displayName":"WRONG_MAP","value":0},{"name":"WRONG_REDUCE","displayName":"WRONG_REDUCE","value":0}]},{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File
System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE:
Number of bytes
read","value":186},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of
bytes written","value":48074},{"name":"FILE_READ_OPS","displayName":"FILE:
Number of read
operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE:
Number of large read
operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of
write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS:
Number of bytes
read","value":0},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of
bytes written
","value":0},{"name":"HDFS_READ_OPS","displayName":"HDFS: Number of read
operations","value":0},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS:
Number of large read
operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of
write
operations","value":0}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce
Framework","counts":[{"name":"COMBINE_INPUT_RECORDS","displayName":"Combine
input
records","value":0},{"name":"COMBINE_OUTPUT_RECORDS","displayName":"Combine
output records","value":0},{"name":"REDUCE_INPUT_GROUPS","displayName":"Reduce
input groups","value":1},{"name":"REDUCE_SHUFFLE_BYTES","displayName":"Reduce
shuffle
bytes","value":120},{"name":"REDUCE_INPUT_RECORDS","displayName":"Reduce input
records","value":10},{"name":"REDUCE_OUTPUT_RECORDS","displayName":"Reduce
output records","value":0},{"name":"SPILLED_RECORDS","displayName":"Spilled
Records","value":10},{"name":"SHUFFLED_MAPS","displayName":"Shuffled Maps
","value
":10},{"name":"FAILED_SHUFFLE","displayName":"Failed
Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map
outputs","value":10},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed
(ms)","value":14},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent
(ms)","value":1070},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical
memory (bytes)
snapshot","value":82780160},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual
memory (bytes)
snapshot","value":714436608},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total
committed heap usage
(bytes)","value":60555264}]},{"name":"org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter","displayName":"File
Output Format Counters
","counts":[{"name":"BYTES_WRITTEN","displayName":"Bytes
Written","value":0}]}]}}}}
