Author: umamahesh
Date: Thu May 22 13:54:53 2014
New Revision: 1596873
URL: http://svn.apache.org/r1596873
Log:
Merge from trunk to fs-encryption branch
Added:
hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestKillAMPreemptionPolicy.java
- copied unchanged from r1596815,
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestKillAMPreemptionPolicy.java
hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_0.23.9-FAILED.jhist
- copied unchanged from r1596815,
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_0.23.9-FAILED.jhist
hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_2.0.3-alpha-FAILED.jhist
- copied unchanged from r1596815,
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_2.0.3-alpha-FAILED.jhist
hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_2.4.0-FAILED.jhist
- copied unchanged from r1596815,
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_2.4.0-FAILED.jhist
Modified:
hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/ (props
changed)
hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/CHANGES.txt
(contents, props changed)
hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/conf/
(props changed)
hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/preemption/KillAMPreemptionPolicy.java
hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr
hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java
hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java
hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
(props changed)
hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
Propchange: hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/
------------------------------------------------------------------------------
Merged
/hadoop/common/branches/HDFS-2006/hadoop-mapreduce-project:r1588992-1596568
Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1595302-1596815
Modified:
hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/CHANGES.txt?rev=1596873&r1=1596872&r2=1596873&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/CHANGES.txt
Thu May 22 13:54:53 2014
@@ -139,6 +139,9 @@ Trunk (Unreleased)
MAPREDUCE-5717. Task pings are interpreted as task progress (jlowe)
+ MAPREDUCE-5867. Fix NPE in KillAMPreemptionPolicy related to
+ ProportionalCapacityPreemptionPolicy (Sunil G via devaraj)
+
Release 2.5.0 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -233,6 +236,9 @@ Release 2.5.0 - UNRELEASED
MAPREDUCE-5814. fat jar with *-default.xml may fail when
mapreduce.job.classloader=true. (Gera Shegalov via jlowe)
+ MAPREDUCE-5309. 2.0.4 JobHistoryParser can't parse certain failed job
+ history files generated by 2.0.3 history server (Rushabh S Shah via jlowe)
+
Release 2.4.1 - UNRELEASED
INCOMPATIBLE CHANGES
Propchange:
hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/CHANGES.txt
------------------------------------------------------------------------------
Merged
/hadoop/common/branches/HDFS-2006/hadoop-mapreduce-project/CHANGES.txt:r1588992-1596568
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1595302-1596815
Propchange: hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/conf/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/conf:r1595302-1596815
Merged
/hadoop/common/branches/HDFS-2006/hadoop-mapreduce-project/conf:r1588992-1596568
Modified:
hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/preemption/KillAMPreemptionPolicy.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/preemption/KillAMPreemptionPolicy.java?rev=1596873&r1=1596872&r2=1596873&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/preemption/KillAMPreemptionPolicy.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/preemption/KillAMPreemptionPolicy.java
Thu May 22 13:54:53 2014
@@ -29,7 +29,9 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.PreemptionContainer;
+import org.apache.hadoop.yarn.api.records.PreemptionContract;
import org.apache.hadoop.yarn.api.records.PreemptionMessage;
+import org.apache.hadoop.yarn.api.records.StrictPreemptionContract;
import org.apache.hadoop.yarn.event.EventHandler;
/**
@@ -52,13 +54,18 @@ public class KillAMPreemptionPolicy impl
public void preempt(Context ctxt, PreemptionMessage preemptionRequests) {
// for both strict and negotiable preemption requests kill the
// container
- for (PreemptionContainer c :
- preemptionRequests.getStrictContract().getContainers()) {
- killContainer(ctxt, c);
+ StrictPreemptionContract strictContract = preemptionRequests
+ .getStrictContract();
+ if (strictContract != null) {
+ for (PreemptionContainer c : strictContract.getContainers()) {
+ killContainer(ctxt, c);
+ }
}
- for (PreemptionContainer c :
- preemptionRequests.getContract().getContainers()) {
- killContainer(ctxt, c);
+ PreemptionContract contract = preemptionRequests.getContract();
+ if (contract != null) {
+ for (PreemptionContainer c : contract.getContainers()) {
+ killContainer(ctxt, c);
+ }
}
}
Modified:
hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr?rev=1596873&r1=1596872&r2=1596873&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr
Thu May 22 13:54:53 2014
@@ -92,11 +92,11 @@
}
},
{"name": "jobQueueName", "type": "string"},
- {"name": "workflowId", "type": "string"},
- {"name": "workflowName", "type": "string"},
- {"name": "workflowNodeName", "type": "string"},
- {"name": "workflowAdjacencies", "type": "string"},
- {"name": "workflowTags", "type": "string"}
+ {"name": "workflowId", "type": ["null","string"], "default": null},
+ {"name": "workflowName", "type": ["null","string"], "default": null},
+ {"name": "workflowNodeName", "type": ["null","string"], "default":
null},
+ {"name": "workflowAdjacencies", "type": ["null","string"],
"default": null},
+ {"name": "workflowTags", "type": ["null","string"], "default": null}
]
},
@@ -136,7 +136,7 @@
{"name": "finishedMaps", "type": "int"},
{"name": "finishedReduces", "type": "int"},
{"name": "jobStatus", "type": "string"},
- {"name": "diagnostics", "type": "string"}
+ {"name": "diagnostics", "type": ["null","string"], "default": null}
]
},
@@ -205,8 +205,8 @@
{"name": "httpPort", "type": "int"},
{"name": "shufflePort", "type": "int"},
{"name": "containerId", "type": "string"},
- {"name": "locality", "type": "string"},
- {"name": "avataar", "type": "string"}
+ {"name": "locality", "type": ["null","string"], "default": null},
+ {"name": "avataar", "type": ["null","string"], "default": null}
]
},
@@ -221,7 +221,7 @@
{"name": "rackname", "type": "string"},
{"name": "status", "type": "string"},
{"name": "error", "type": "string"},
- {"name": "counters", "type": "JhCounters"},
+ {"name": "counters", "type": ["null","JhCounters"], "default": null},
{"name": "clockSplits", "type": { "type": "array", "items": "int"}},
{"name": "cpuUsages", "type": { "type": "array", "items": "int"}},
{"name": "vMemKbytes", "type": { "type": "array", "items": "int"}},
@@ -237,7 +237,7 @@
{"name": "error", "type": "string"},
{"name": "failedDueToAttempt", "type": ["null", "string"] },
{"name": "status", "type": "string"},
- {"name": "counters", "type": "JhCounters"}
+ {"name": "counters", "type": ["null","JhCounters"], "default": null}
]
},
@@ -248,7 +248,7 @@
{"name": "finishTime", "type": "long"},
{"name": "status", "type": "string"},
{"name": "counters", "type": "JhCounters"},
- {"name": "successfulAttemptId", "type": "string"}
+ {"name": "successfulAttemptId", "type": ["null","string"],
"default": null}
]
},
Modified:
hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java?rev=1596873&r1=1596872&r2=1596873&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java
Thu May 22 13:54:53 2014
@@ -35,6 +35,7 @@ import org.apache.avro.Schema;
import org.apache.avro.io.Decoder;
import org.apache.avro.io.DecoderFactory;
import org.apache.avro.io.DatumReader;
+import org.apache.avro.specific.SpecificData;
import org.apache.avro.specific.SpecificDatumReader;
@InterfaceAudience.Private
@@ -69,9 +70,10 @@ public class EventReader implements Clos
if (!EventWriter.VERSION.equals(version)) {
throw new IOException("Incompatible event log version: "+version);
}
-
+
+ Schema myschema = new
SpecificData(Event.class.getClassLoader()).getSchema(Event.class);
this.schema = Schema.parse(in.readLine());
- this.reader = new SpecificDatumReader(schema);
+ this.reader = new SpecificDatumReader(schema, myschema);
this.decoder = DecoderFactory.get().jsonDecoder(schema, in);
}
@@ -173,13 +175,15 @@ public class EventReader implements Clos
static Counters fromAvro(JhCounters counters) {
Counters result = new Counters();
- for (JhCounterGroup g : counters.groups) {
- CounterGroup group =
- result.addGroup(StringInterner.weakIntern(g.name.toString()),
- StringInterner.weakIntern(g.displayName.toString()));
- for (JhCounter c : g.counts) {
- group.addCounter(StringInterner.weakIntern(c.name.toString()),
- StringInterner.weakIntern(c.displayName.toString()), c.value);
+ if(counters != null) {
+ for (JhCounterGroup g : counters.groups) {
+ CounterGroup group =
+ result.addGroup(StringInterner.weakIntern(g.name.toString()),
+ StringInterner.weakIntern(g.displayName.toString()));
+ for (JhCounter c : g.counts) {
+ group.addCounter(StringInterner.weakIntern(c.name.toString()),
+ StringInterner.weakIntern(c.displayName.toString()), c.value);
+ }
}
}
return result;
Modified:
hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java?rev=1596873&r1=1596872&r2=1596873&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java
Thu May 22 13:54:53 2014
@@ -288,8 +288,18 @@ public class JobHistoryParser implements
private void handleTaskAttemptFailedEvent(
TaskAttemptUnsuccessfulCompletionEvent event) {
TaskInfo taskInfo = info.tasksMap.get(event.getTaskId());
+ if(taskInfo == null) {
+ LOG.warn("TaskInfo is null for TaskAttemptUnsuccessfulCompletionEvent"
+ + " taskId: " + event.getTaskId().toString());
+ return;
+ }
TaskAttemptInfo attemptInfo =
taskInfo.attemptsMap.get(event.getTaskAttemptId());
+ if(attemptInfo == null) {
+ LOG.warn("AttemptInfo is null for TaskAttemptUnsuccessfulCompletionEvent"
+ + " taskAttemptId: " + event.getTaskAttemptId().toString());
+ return;
+ }
attemptInfo.finishTime = event.getFinishTime();
attemptInfo.error = StringInterner.weakIntern(event.getError());
attemptInfo.status = StringInterner.weakIntern(event.getTaskStatus());
Propchange:
hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
------------------------------------------------------------------------------
Merged
/hadoop/common/branches/HDFS-2006/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:r1588992-1596568
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:r1595302-1596815
Modified:
hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java?rev=1596873&r1=1596872&r2=1596873&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
Thu May 22 13:54:53 2014
@@ -891,4 +891,58 @@ public class TestJobHistoryParsing {
fsdis.close();
}
}
-}
+
+ /**
+ * Test compatibility of JobHistoryParser with 2.0.3-alpha history files
+ * @throws IOException
+ */
+ @Test
+ public void testTaskAttemptUnsuccessfulCompletionWithoutCounters203() throws
IOException
+ {
+ Path histPath = new Path(getClass().getClassLoader().getResource(
+ "job_2.0.3-alpha-FAILED.jhist").getFile());
+ JobHistoryParser parser = new JobHistoryParser(FileSystem.getLocal
+ (new Configuration()), histPath);
+ JobInfo jobInfo = parser.parse();
+ LOG.info(" job info: " + jobInfo.getJobname() + " "
+ + jobInfo.getFinishedMaps() + " "
+ + jobInfo.getTotalMaps() + " "
+ + jobInfo.getJobId() ) ;
+ }
+
+ /**
+ * Test compatibility of JobHistoryParser with 2.4.0 history files
+ * @throws IOException
+ */
+ @Test
+ public void testTaskAttemptUnsuccessfulCompletionWithoutCounters240() throws
IOException
+ {
+ Path histPath = new Path(getClass().getClassLoader().getResource(
+ "job_2.4.0-FAILED.jhist").getFile());
+ JobHistoryParser parser = new JobHistoryParser(FileSystem.getLocal
+ (new Configuration()), histPath);
+ JobInfo jobInfo = parser.parse();
+ LOG.info(" job info: " + jobInfo.getJobname() + " "
+ + jobInfo.getFinishedMaps() + " "
+ + jobInfo.getTotalMaps() + " "
+ + jobInfo.getJobId() );
+ }
+
+ /**
+ * Test compatibility of JobHistoryParser with 0.23.9 history files
+ * @throws IOException
+ */
+ @Test
+ public void testTaskAttemptUnsuccessfulCompletionWithoutCounters0239()
throws IOException
+ {
+ Path histPath = new Path(getClass().getClassLoader().getResource(
+ "job_0.23.9-FAILED.jhist").getFile());
+ JobHistoryParser parser = new JobHistoryParser(FileSystem.getLocal
+ (new Configuration()), histPath);
+ JobInfo jobInfo = parser.parse();
+ LOG.info(" job info: " + jobInfo.getJobname() + " "
+ + jobInfo.getFinishedMaps() + " "
+ + jobInfo.getTotalMaps() + " "
+ + jobInfo.getJobId() ) ;
+ }
+}
\ No newline at end of file
