svn commit: r1379585 - in /hadoop/common/branches/branch-2/hadoop-mapreduce-project: CHANGES.txt hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistor
Author: bobby
Date: Fri Aug 31 20:26:26 2012
New Revision: 1379585
URL: http://svn.apache.org/viewvc?rev=1379585view=rev
Log:
svn merge -c 1379584 FIXES: MAPREDUCE-4612. job summary file permissions not
set when its created (tgraves via bobby)
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1379585r1=1379584r2=1379585view=diff
==
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Fri
Aug 31 20:26:26 2012
@@ -185,6 +185,9 @@ Release 2.1.0-alpha - Unreleased
MAPREDUCE-4380. Empty Userlogs directory is getting created under logs
directory (Devaraj K via bobby)
+MAPREDUCE-4612. job summary file permissions not set when its created
+(tgraves via bobby)
+
Release 2.0.0-alpha - 05-23-2012
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java?rev=1379585r1=1379584r2=1379585view=diff
==
---
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
Fri Aug 31 20:26:26 2012
@@ -661,6 +661,8 @@ public class JobHistoryEventHandler exte
summaryFileOut = doneDirFS.create(qualifiedSummaryDoneFile, true);
summaryFileOut.writeUTF(mi.getJobSummary().getJobSummaryString());
summaryFileOut.close();
+ doneDirFS.setPermission(qualifiedSummaryDoneFile, new FsPermission(
+ JobHistoryUtils.HISTORY_INTERMEDIATE_FILE_PERMISSIONS));
} catch (IOException e) {
LOG.info(Unable to write out JobSummaryInfo to [
+ qualifiedSummaryDoneFile + ], e);
@@ -894,7 +896,7 @@ public class JobHistoryEventHandler exte
stagingDirFS.delete(fromPath, false);
}
-}
+ }
boolean pathExists(FileSystem fileSys, Path path) throws IOException {
return fileSys.exists(path);
svn commit: r1379595 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/ hadoop-mapreduce-client/hadoop-m
Author: bobby
Date: Fri Aug 31 20:37:41 2012
New Revision: 1379595
URL: http://svn.apache.org/viewvc?rev=1379595view=rev
Log:
MAPREDUCE-4614. Simplify debugging a job's tokens (daryn via bobby)
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1379595r1=1379594r2=1379595view=diff
==
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Fri Aug 31
20:37:41 2012
@@ -323,9 +323,6 @@ Release 2.1.0-alpha - Unreleased
MAPREDUCE-4380. Empty Userlogs directory is getting created under logs
directory (Devaraj K via bobby)
-MAPREDUCE-4612. job summary file permissions not set when its created
-(tgraves via bobby)
-
Release 2.0.0-alpha - 05-23-2012
INCOMPATIBLE CHANGES
@@ -856,6 +853,11 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4600. TestTokenCache.java from MRV1 no longer compiles (daryn
via bobby)
+MAPREDUCE-4612. job summary file permissions not set when its created
+(tgraves via bobby)
+
+MAPREDUCE-4614. Simplify debugging a job's tokens (daryn via bobby)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java?rev=1379595r1=1379594r2=1379595view=diff
==
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
Fri Aug 31 20:37:41 2012
@@ -93,7 +93,11 @@ class YarnChild {
// Security framework already loaded the tokens into current ugi
Credentials credentials =
UserGroupInformation.getCurrentUser().getCredentials();
-
+LOG.info(Executing with tokens:);
+for (Token? token: credentials.getAllTokens()) {
+ LOG.info(token);
+}
+
// Create TaskUmbilicalProtocol as actual task owner.
UserGroupInformation taskOwner =
UserGroupInformation.createRemoteUser(firstTaskid.getJobID().toString());
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java?rev=1379595r1=1379594r2=1379595view=diff
==
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
Fri Aug 31 20:37:41 2012
@@ -435,13 +435,9 @@ class JobSubmitter {
private void printTokens(JobID jobId,
Credentials credentials) throws IOException {
-if (LOG.isDebugEnabled()) {
- LOG.debug(Printing tokens for job: + jobId);
- for(Token? token: credentials.getAllTokens()) {
-if (token.getKind().toString().equals(HDFS_DELEGATION_TOKEN)) {
- LOG.debug(Submitting with + token);
-}
- }
+LOG.info(Submitting tokens for job: + jobId);
+for (Token? token: credentials.getAllTokens()) {
+ LOG.info(token);
}
}
svn commit: r1379596 - in /hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/ hadoop-mapreduce-cli
Author: bobby
Date: Fri Aug 31 20:39:03 2012
New Revision: 1379596
URL: http://svn.apache.org/viewvc?rev=1379596view=rev
Log:
svn merge -c 1379595 FIXES: MAPREDUCE-4614. Simplify debugging a job's tokens
(daryn via bobby)
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1379596r1=1379595r2=1379596view=diff
==
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Fri
Aug 31 20:39:03 2012
@@ -185,9 +185,6 @@ Release 2.1.0-alpha - Unreleased
MAPREDUCE-4380. Empty Userlogs directory is getting created under logs
directory (Devaraj K via bobby)
-MAPREDUCE-4612. job summary file permissions not set when its created
-(tgraves via bobby)
-
Release 2.0.0-alpha - 05-23-2012
INCOMPATIBLE CHANGES
@@ -732,6 +729,11 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4600. TestTokenCache.java from MRV1 no longer compiles (daryn
via bobby)
+MAPREDUCE-4612. job summary file permissions not set when its created
+(tgraves via bobby)
+
+MAPREDUCE-4614. Simplify debugging a job's tokens (daryn via bobby)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java?rev=1379596r1=1379595r2=1379596view=diff
==
---
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
Fri Aug 31 20:39:03 2012
@@ -93,7 +93,11 @@ class YarnChild {
// Security framework already loaded the tokens into current ugi
Credentials credentials =
UserGroupInformation.getCurrentUser().getCredentials();
-
+LOG.info(Executing with tokens:);
+for (Token? token: credentials.getAllTokens()) {
+ LOG.info(token);
+}
+
// Create TaskUmbilicalProtocol as actual task owner.
UserGroupInformation taskOwner =
UserGroupInformation.createRemoteUser(firstTaskid.getJobID().toString());
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java?rev=1379596r1=1379595r2=1379596view=diff
==
---
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
Fri Aug 31 20:39:03 2012
@@ -435,13 +435,9 @@ class JobSubmitter {
private void printTokens(JobID jobId,
Credentials credentials) throws IOException {
-if (LOG.isDebugEnabled()) {
- LOG.debug(Printing tokens for job: + jobId);
- for(Token? token: credentials.getAllTokens()) {
-if (token.getKind().toString().equals(HDFS_DELEGATION_TOKEN)) {
- LOG.debug(Submitting with + token);
-}
- }
+LOG.info(Submitting tokens for job: + jobId);
+for (Token? token: credentials.getAllTokens()) {
+ LOG.info(token);
}
}
svn commit: r1379597 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/ hadoop-mapreduce-
Author: bobby
Date: Fri Aug 31 20:41:23 2012
New Revision: 1379597
URL: http://svn.apache.org/viewvc?rev=1379597view=rev
Log:
MAPREDUCE-4614. Simplify debugging a job's tokens (daryn via bobby)
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1379597r1=1379596r2=1379597view=diff
==
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Fri
Aug 31 20:41:23 2012
@@ -424,6 +424,8 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4612. job summary file permissions not set when its created
(tgraves via bobby)
+MAPREDUCE-4614. Simplify debugging a job's tokens (daryn via bobby)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java?rev=1379597r1=1379596r2=1379597view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
Fri Aug 31 20:41:23 2012
@@ -93,7 +93,11 @@ class YarnChild {
// Security framework already loaded the tokens into current ugi
Credentials credentials =
UserGroupInformation.getCurrentUser().getCredentials();
-
+LOG.info(Executing with tokens:);
+for (Token? token: credentials.getAllTokens()) {
+ LOG.info(token);
+}
+
// Create TaskUmbilicalProtocol as actual task owner.
UserGroupInformation taskOwner =
UserGroupInformation.createRemoteUser(firstTaskid.getJobID().toString());
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java?rev=1379597r1=1379596r2=1379597view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
Fri Aug 31 20:41:23 2012
@@ -428,12 +428,9 @@ class JobSubmitter {
private void printTokens(JobID jobId,
Credentials credentials) throws IOException {
if (LOG.isDebugEnabled()) {
- LOG.debug(Printing tokens for job: + jobId);
- for(Token? token: credentials.getAllTokens()) {
-if (token.getKind().toString().equals(HDFS_DELEGATION_TOKEN)) {
- LOG.debug(Submitting with +
-
org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier.stringifyToken(token));
-}
+ LOG.info(Submitting tokens for job: + jobId);
+ for (Token? token: credentials.getAllTokens()) {
+LOG.info(token);
}
}
}
svn commit: r1379599 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ hadoop-mapreduce-c
Author: tgraves
Date: Fri Aug 31 20:43:46 2012
New Revision: 1379599
URL: http://svn.apache.org/viewvc?rev=1379599view=rev
Log:
MAPREDUCE-4611. MR AM dies badly when Node is decommissioned (Robert Evans via
tgraves)
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1379599r1=1379598r2=1379599view=diff
==
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Fri Aug 31
20:43:46 2012
@@ -858,6 +858,9 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4614. Simplify debugging a job's tokens (daryn via bobby)
+MAPREDUCE-4611. MR AM dies badly when Node is decommissioned (Robert
+Evans via tgraves)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java?rev=1379599r1=1379598r2=1379599view=diff
==
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
Fri Aug 31 20:43:46 2012
@@ -99,8 +99,8 @@ public class JobHistoryEventHandler exte
protected static final MapJobId, MetaInfo fileMap =
Collections.JobId,MetaInfosynchronizedMap(new HashMapJobId,MetaInfo());
- // Has a signal (SIGTERM etc) been issued?
- protected volatile boolean isSignalled = false;
+ // should job completion be force when the AM shuts down?
+ protected volatile boolean forceJobCompletion = false;
public JobHistoryEventHandler(AppContext context, int startCount) {
super(JobHistoryEventHandler);
@@ -322,7 +322,7 @@ public class JobHistoryEventHandler exte
// Process JobUnsuccessfulCompletionEvent for jobIds which still haven't
// closed their event writers
IteratorJobId jobIt = fileMap.keySet().iterator();
-if(isSignalled) {
+if(forceJobCompletion) {
while (jobIt.hasNext()) {
JobId toClose = jobIt.next();
MetaInfo mi = fileMap.get(toClose);
@@ -911,9 +911,9 @@ public class JobHistoryEventHandler exte
return tmpFileName.substring(0, tmpFileName.length()-4);
}
- public void setSignalled(boolean isSignalled) {
-this.isSignalled = isSignalled;
-LOG.info(JobHistoryEventHandler notified that isSignalled was
- + isSignalled);
+ public void setForcejobCompletion(boolean forceJobCompletion) {
+this.forceJobCompletion = forceJobCompletion;
+LOG.info(JobHistoryEventHandler notified that forceJobCompletion is
+ + forceJobCompletion);
}
}
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java?rev=1379599r1=1379598r2=1379599view=diff
==
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
(original)
+++
svn commit: r1379602 - in /hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ hadoop
Author: tgraves
Date: Fri Aug 31 20:45:34 2012
New Revision: 1379602
URL: http://svn.apache.org/viewvc?rev=1379602view=rev
Log:
merge -r 1379598:1379599 from trunk. FIXES: MAPREDUCE-4611
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1379602r1=1379601r2=1379602view=diff
==
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Fri
Aug 31 20:45:34 2012
@@ -734,6 +734,9 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4614. Simplify debugging a job's tokens (daryn via bobby)
+MAPREDUCE-4611. MR AM dies badly when Node is decommissioned (Robert
+Evans via tgraves)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java?rev=1379602r1=1379601r2=1379602view=diff
==
---
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
Fri Aug 31 20:45:34 2012
@@ -99,8 +99,8 @@ public class JobHistoryEventHandler exte
protected static final MapJobId, MetaInfo fileMap =
Collections.JobId,MetaInfosynchronizedMap(new HashMapJobId,MetaInfo());
- // Has a signal (SIGTERM etc) been issued?
- protected volatile boolean isSignalled = false;
+ // should job completion be force when the AM shuts down?
+ protected volatile boolean forceJobCompletion = false;
public JobHistoryEventHandler(AppContext context, int startCount) {
super(JobHistoryEventHandler);
@@ -322,7 +322,7 @@ public class JobHistoryEventHandler exte
// Process JobUnsuccessfulCompletionEvent for jobIds which still haven't
// closed their event writers
IteratorJobId jobIt = fileMap.keySet().iterator();
-if(isSignalled) {
+if(forceJobCompletion) {
while (jobIt.hasNext()) {
JobId toClose = jobIt.next();
MetaInfo mi = fileMap.get(toClose);
@@ -911,9 +911,9 @@ public class JobHistoryEventHandler exte
return tmpFileName.substring(0, tmpFileName.length()-4);
}
- public void setSignalled(boolean isSignalled) {
-this.isSignalled = isSignalled;
-LOG.info(JobHistoryEventHandler notified that isSignalled was
- + isSignalled);
+ public void setForcejobCompletion(boolean forceJobCompletion) {
+this.forceJobCompletion = forceJobCompletion;
+LOG.info(JobHistoryEventHandler notified that forceJobCompletion is
+ + forceJobCompletion);
}
}
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java?rev=1379602r1=1379601r2=1379602view=diff
==
---
svn commit: r1379604 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ had
Author: tgraves
Date: Fri Aug 31 20:48:22 2012
New Revision: 1379604
URL: http://svn.apache.org/viewvc?rev=1379604view=rev
Log:
merge -r 1379601:1379602 from branch-2. FIXES: MAPREDUCE-4611
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1379604r1=1379603r2=1379604view=diff
==
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Fri
Aug 31 20:48:22 2012
@@ -426,6 +426,9 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4614. Simplify debugging a job's tokens (daryn via bobby)
+MAPREDUCE-4611. MR AM dies badly when Node is decommissioned (Robert
+Evans via tgraves)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java?rev=1379604r1=1379603r2=1379604view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
Fri Aug 31 20:48:22 2012
@@ -99,8 +99,8 @@ public class JobHistoryEventHandler exte
protected static final MapJobId, MetaInfo fileMap =
Collections.JobId,MetaInfosynchronizedMap(new HashMapJobId,MetaInfo());
- // Has a signal (SIGTERM etc) been issued?
- protected volatile boolean isSignalled = false;
+ // should job completion be force when the AM shuts down?
+ protected volatile boolean forceJobCompletion = false;
public JobHistoryEventHandler(AppContext context, int startCount) {
super(JobHistoryEventHandler);
@@ -322,7 +322,7 @@ public class JobHistoryEventHandler exte
// Process JobUnsuccessfulCompletionEvent for jobIds which still haven't
// closed their event writers
IteratorJobId jobIt = fileMap.keySet().iterator();
-if(isSignalled) {
+if(forceJobCompletion) {
while (jobIt.hasNext()) {
JobId toClose = jobIt.next();
MetaInfo mi = fileMap.get(toClose);
@@ -911,9 +911,9 @@ public class JobHistoryEventHandler exte
return tmpFileName.substring(0, tmpFileName.length()-4);
}
- public void setSignalled(boolean isSignalled) {
-this.isSignalled = isSignalled;
-LOG.info(JobHistoryEventHandler notified that isSignalled was
- + isSignalled);
+ public void setForcejobCompletion(boolean forceJobCompletion) {
+this.forceJobCompletion = forceJobCompletion;
+LOG.info(JobHistoryEventHandler notified that forceJobCompletion is
+ + forceJobCompletion);
}
}
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java?rev=1379604r1=1379603r2=1379604view=diff
==
---
svn commit: r1379617 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ hadoop-mapreduce-client/hadoop-
Author: jeagles
Date: Fri Aug 31 21:25:25 2012
New Revision: 1379617
URL: http://svn.apache.org/viewvc?rev=1379617view=rev
Log:
MAPREDUCE-4604. In mapred-default, mapreduce.map.maxattempts
mapreduce.reduce.maxattempts defaults are set to 4 as well as
mapreduce.job.maxtaskfailures.per.tracker. (Ravi Prakash via jeagles)
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestJobConf.java
hadoop/common/trunk/hadoop-mapreduce-project/src/java/mapred-default.xml
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1379617r1=1379616r2=1379617view=diff
==
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Fri Aug 31
21:25:25 2012
@@ -861,6 +861,10 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4611. MR AM dies badly when Node is decommissioned (Robert
Evans via tgraves)
+MAPREDUCE-4604. In mapred-default, mapreduce.map.maxattempts
+mapreduce.reduce.maxattempts defaults are set to 4 as well as
+mapreduce.job.maxtaskfailures.per.tracker. (Ravi Prakash via jeagles)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java?rev=1379617r1=1379616r2=1379617view=diff
==
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
Fri Aug 31 21:25:25 2012
@@ -1357,7 +1357,7 @@ public class JobConf extends Configurati
* @return the maximum no. of failures of a given job per tasktracker.
*/
public int getMaxTaskFailuresPerTracker() {
-return getInt(JobContext.MAX_TASK_FAILURES_PER_TRACKER, 4);
+return getInt(JobContext.MAX_TASK_FAILURES_PER_TRACKER, 3);
}
/**
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml?rev=1379617r1=1379616r2=1379617view=diff
==
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
Fri Aug 31 21:25:25 2012
@@ -797,9 +797,12 @@
property
namemapreduce.job.maxtaskfailures.per.tracker/name
- value4/value
+ value3/value
descriptionThe number of task-failures on a tasktracker of a given job
- after which new tasks of that job aren't assigned to it.
+ after which new tasks of that job aren't assigned to it. It
+ MUST be less than mapreduce.map.maxattempts and
+ mapreduce.reduce.maxattempts otherwise the failed task will
+ never be tried on a different node.
/description
/property
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestJobConf.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestJobConf.java?rev=1379617r1=1379616r2=1379617view=diff
==
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestJobConf.java
(original)
+++
svn commit: r1379621 - in /hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ hadoop-ma
Author: jeagles
Date: Fri Aug 31 21:30:19 2012
New Revision: 1379621
URL: http://svn.apache.org/viewvc?rev=1379621view=rev
Log:
svn merge -c 1379617 from trunk FIXES: MAPREDUCE-4604. In mapred-default,
mapreduce.map.maxattempts mapreduce.reduce.maxattempts defaults are set to 4
as well as mapreduce.job.maxtaskfailures.per.tracker. (Ravi Prakash via jeagles)
Modified:
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestJobConf.java
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/src/java/mapred-default.xml
Modified:
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/CHANGES.txt?rev=1379621r1=1379620r2=1379621view=diff
==
---
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/CHANGES.txt
Fri Aug 31 21:30:19 2012
@@ -599,6 +599,10 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4600. TestTokenCache.java from MRV1 no longer compiles (daryn
via bobby)
+MAPREDUCE-4604. In mapred-default, mapreduce.map.maxattempts
+mapreduce.reduce.maxattempts defaults are set to 4 as well as
+mapreduce.job.maxtaskfailures.per.tracker. (Ravi Prakash via jeagles)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java?rev=1379621r1=1379620r2=1379621view=diff
==
---
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
(original)
+++
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
Fri Aug 31 21:30:19 2012
@@ -1357,7 +1357,7 @@ public class JobConf extends Configurati
* @return the maximum no. of failures of a given job per tasktracker.
*/
public int getMaxTaskFailuresPerTracker() {
-return getInt(JobContext.MAX_TASK_FAILURES_PER_TRACKER, 4);
+return getInt(JobContext.MAX_TASK_FAILURES_PER_TRACKER, 3);
}
/**
Modified:
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml?rev=1379621r1=1379620r2=1379621view=diff
==
---
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
(original)
+++
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
Fri Aug 31 21:30:19 2012
@@ -782,9 +782,12 @@
property
namemapreduce.job.maxtaskfailures.per.tracker/name
- value4/value
+ value3/value
descriptionThe number of task-failures on a tasktracker of a given job
- after which new tasks of that job aren't assigned to it.
+ after which new tasks of that job aren't assigned to it. It
+ MUST be less than mapreduce.map.maxattempts and
+ mapreduce.reduce.maxattempts otherwise the failed task will
+ never be tried on a different node.
/description
/property
Modified:
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestJobConf.java
URL:
svn commit: r1379625 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ hadoop-mapreduce
Author: jeagles
Date: Fri Aug 31 21:32:14 2012
New Revision: 1379625
URL: http://svn.apache.org/viewvc?rev=1379625view=rev
Log:
svn merge -c 1379617 from trunk FIXES: MAPREDUCE-4604. In mapred-default,
mapreduce.map.maxattempts mapreduce.reduce.maxattempts defaults are set to 4
as well as mapreduce.job.maxtaskfailures.per.tracker. (Ravi Prakash via jeagles)
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestJobConf.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/java/mapred-default.xml
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1379625r1=1379624r2=1379625view=diff
==
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Fri
Aug 31 21:32:14 2012
@@ -429,6 +429,10 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4611. MR AM dies badly when Node is decommissioned (Robert
Evans via tgraves)
+MAPREDUCE-4604. In mapred-default, mapreduce.map.maxattempts
+mapreduce.reduce.maxattempts defaults are set to 4 as well as
+mapreduce.job.maxtaskfailures.per.tracker. (Ravi Prakash via jeagles)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java?rev=1379625r1=1379624r2=1379625view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
Fri Aug 31 21:32:14 2012
@@ -1357,7 +1357,7 @@ public class JobConf extends Configurati
* @return the maximum no. of failures of a given job per tasktracker.
*/
public int getMaxTaskFailuresPerTracker() {
-return getInt(JobContext.MAX_TASK_FAILURES_PER_TRACKER, 4);
+return getInt(JobContext.MAX_TASK_FAILURES_PER_TRACKER, 3);
}
/**
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml?rev=1379625r1=1379624r2=1379625view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
Fri Aug 31 21:32:14 2012
@@ -782,9 +782,12 @@
property
namemapreduce.job.maxtaskfailures.per.tracker/name
- value4/value
+ value3/value
descriptionThe number of task-failures on a tasktracker of a given job
- after which new tasks of that job aren't assigned to it.
+ after which new tasks of that job aren't assigned to it. It
+ MUST be less than mapreduce.map.maxattempts and
+ mapreduce.reduce.maxattempts otherwise the failed task will
+ never be tried on a different node.
/description
/property
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestJobConf.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestJobConf.java?rev=1379625r1=1379624r2=1379625view=diff
svn commit: r1379650 - in /hadoop/common/branches/MR-3902/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapreduce/v2/app2/ hadoop-ma
Author: sseth
Date: Fri Aug 31 22:39:10 2012
New Revision: 1379650
URL: http://svn.apache.org/viewvc?rev=1379650view=rev
Log:
MAPREDUCE-4625. Statistics logging in the AM scheduler.
Modified:
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/CHANGES.txt.MR-3902
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapreduce/v2/app2/MRAppMaster.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapreduce/v2/app2/job/event/TaskAttemptEventType.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapreduce/v2/app2/job/impl/TaskImpl.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapreduce/v2/app2/rm/AMSchedulerEventContainersAllocated.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapreduce/v2/app2/rm/AMSchedulerEventType.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapreduce/v2/app2/rm/RMContainerAllocator.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapreduce/v2/app2/rm/RMContainerRequestor.java
Modified:
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/CHANGES.txt.MR-3902
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-mapreduce-project/CHANGES.txt.MR-3902?rev=1379650r1=1379649r2=1379650view=diff
==
--- hadoop/common/branches/MR-3902/hadoop-mapreduce-project/CHANGES.txt.MR-3902
(original)
+++ hadoop/common/branches/MR-3902/hadoop-mapreduce-project/CHANGES.txt.MR-3902
Fri Aug 31 22:39:10 2012
@@ -6,3 +6,5 @@ Branch MR-3902
MAPREDUCE-4599. Prevent contianer launches on blacklisted hosts. (Tsuyoshi
OZAWA via sseth)
MAPREDUCE-4609. RMContainerAllocator scheduler interval should be
configurable. (Tsuyoshi OZAWA via sseth)
+
+ MAPREDUCE-4625. Statistics logging in the AM scheduler. (sseth)
Modified:
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapreduce/v2/app2/MRAppMaster.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapreduce/v2/app2/MRAppMaster.java?rev=1379650r1=1379649r2=1379650view=diff
==
---
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapreduce/v2/app2/MRAppMaster.java
(original)
+++
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapreduce/v2/app2/MRAppMaster.java
Fri Aug 31 22:39:10 2012
@@ -444,6 +444,7 @@ public class MRAppMaster extends Composi
// second timeout before the exit.
// TODO XXX: Modify TaskAttemptCleaner to empty it's queue while stopping.
public void handle(JobFinishEvent event) {
+ LOG.info(Handling JobFinished Event);
AMShutdownRunnable r = new AMShutdownRunnable();
Thread t = new Thread(r, AMShutdownThread);
t.start();
@@ -506,6 +507,8 @@ public class MRAppMaster extends Composi
@Override
public void run() {
maybeSendJobEndNotification();
+// TODO XXX Add a timeout.
+LOG.info(Waiting for all containers and TaskAttempts to complete);
while (!allContainersComplete() || !allTaskAttemptsComplete()) {
try {
synchronized(this) {
@@ -516,6 +519,7 @@ public class MRAppMaster extends Composi
break;
}
}
+LOG.info(All Containers and TaskAttempts Complete. Stopping
services);
stopAM();
LOG.info(AM Shutdown Thread Completing);
}
Modified:
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapreduce/v2/app2/job/event/TaskAttemptEventType.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapreduce/v2/app2/job/event/TaskAttemptEventType.java?rev=1379650r1=1379649r2=1379650view=diff
==
---
svn commit: r1379654 - in /hadoop/common/branches/MR-3902/hadoop-mapreduce-project: CHANGES.txt.MR-3902 hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapreduce/v
Author: sseth
Date: Fri Aug 31 22:41:45 2012
New Revision: 1379654
URL: http://svn.apache.org/viewvc?rev=1379654view=rev
Log:
MAPREDUCE-4624. Reduce scheduling fixes, factor in MR-4437 (sseth)
Modified:
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/CHANGES.txt.MR-3902
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapreduce/v2/app2/rm/RMContainerAllocator.java
Modified:
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/CHANGES.txt.MR-3902
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-mapreduce-project/CHANGES.txt.MR-3902?rev=1379654r1=1379653r2=1379654view=diff
==
--- hadoop/common/branches/MR-3902/hadoop-mapreduce-project/CHANGES.txt.MR-3902
(original)
+++ hadoop/common/branches/MR-3902/hadoop-mapreduce-project/CHANGES.txt.MR-3902
Fri Aug 31 22:41:45 2012
@@ -8,3 +8,5 @@ Branch MR-3902
MAPREDUCE-4609. RMContainerAllocator scheduler interval should be
configurable. (Tsuyoshi OZAWA via sseth)
MAPREDUCE-4625. Statistics logging in the AM scheduler. (sseth)
+
+ MAPREDUCE-4624. Reduce scheduling fixes, factor in MR-4437. (sseth)
Modified:
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapreduce/v2/app2/rm/RMContainerAllocator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapreduce/v2/app2/rm/RMContainerAllocator.java?rev=1379654r1=1379653r2=1379654view=diff
==
---
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapreduce/v2/app2/rm/RMContainerAllocator.java
(original)
+++
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapreduce/v2/app2/rm/RMContainerAllocator.java
Fri Aug 31 22:41:45 2012
@@ -123,6 +123,9 @@ public class RMContainerAllocator extend
Timer scheduleTimer;
ScheduleTimerTask scheduleTimerTask;
private long lastScheduleTime = 0l;
+ private int lastCompletedTasks = 0;
+ private int completedMaps = 0;
+ private int completedReduces = 0;
/*
Vocabulary Used:
@@ -287,13 +290,9 @@ public class RMContainerAllocator extend
@Override
public void run() {
- // TODO XXX XXX: Reduces are not being shceduled. Forcing them via this
for now. Figure out when reduce schedule should be recomputed.
- // TODO XXX. Does this need to be stopped before the service stop()
if (clock.getTime() - lastScheduleTime scheduleInterval shouldRun) {
handle(new AMSchedulerEventContainersAllocated(
-Collections.ContainerId emptyList(), true));
-// Sending a false. Just try to flush available containers.
-// The decision to schedule reduces may need to be based on available
containers.
+Collections.ContainerId emptyList(), false));
}
}
@@ -330,13 +329,12 @@ public class RMContainerAllocator extend
}
protected synchronized void handleEvent(AMSchedulerEvent sEvent) {
-
+// Recalculating reduce schedule here since it's required for most events.
+recalculateReduceSchedule = true;
LOG.info(XXX: Processing the event + sEvent.toString());
switch(sEvent.getType()) {
-// TODO XXX: recalculateReduceSchedule may need to bet set on other events
- not just containerAllocated.
case S_TA_LAUNCH_REQUEST:
handleTaLaunchRequest((AMSchedulerTALaunchRequestEvent) sEvent);
- // Add to queue of pending tasks.
break;
case S_TA_STOP_REQUEST: //Effectively means a failure.
handleTaStopRequest((AMSchedulerTAStopRequestEvent)sEvent);
@@ -413,11 +411,6 @@ public class RMContainerAllocator extend
}
private void handleTaSucceededRequest(AMSchedulerTASucceededEvent event) {
-// TODO XXX Part of re-use.
-// TODO XXX Also may change after state machines are finalized.
-// XXX: Maybe send the request to the task before sending it to the
scheduler - the scheduler can then
-// query the task to figure out whether the taskAttempt is the
successfulAttempt - and whether to count it towards the reduce ramp up.
-// Otherwise - Job.getCompletedMaps() - will give an out of date
picture, since the scheduler event will always be generated before the
TaskCompleted event to the job.
attemptToLaunchRequestMap.remove(event.getAttemptID());
ContainerId containerId = assignedRequests.remove(event.getAttemptID());
if (containerId != null) { // TODO Should not be null. Confirm.
@@ -435,15 +428,20 @@ public class RMContainerAllocator
