Author: arp
Date: Mon Sep 16 20:59:02 2013
New Revision: 1523804
URL: http://svn.apache.org/r1523804
Log:
Merging r1523402 through r1523803 from trunk to branch HDFS-2832
Modified:
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/ (props changed)
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/CHANGES.txt
(contents, props changed)
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/conf/ (props
changed)
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/MRClientProtocol.proto
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_protos.proto
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_service_protos.proto
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
(contents, props changed)
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMerger.java
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/proto/HSAdminRefreshProtocol.proto
Propchange: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1523402-1523803
Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/CHANGES.txt?rev=1523804&r1=1523803&r2=1523804&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/CHANGES.txt Mon
Sep 16 20:59:02 2013
@@ -190,6 +190,8 @@ Release 2.1.1-beta - UNRELEASED
but just before ClientService to avoid race conditions during RM restart.
(Jian He via vinodkv)
+ MAPREDUCE-5379. Include token tracking ids in jobconf. (kkambatl via tucu)
+
OPTIMIZATIONS
MAPREDUCE-5446. TestJobHistoryEvents and TestJobHistoryParsing have race
@@ -265,6 +267,9 @@ Release 2.1.1-beta - UNRELEASED
MAPREDUCE-5164. mapred job and queue commands omit HADOOP_CLIENT_OPTS
(Nemon Lou via devaraj)
+ MAPREDUCE-5493. Cleanup in-memory & on-disk segments to prevent leak on
+ shuffle completion. (jlowe via acmurthy)
+
Release 2.1.0-beta - 2013-08-22
INCOMPATIBLE CHANGES
Propchange:
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/CHANGES.txt
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1523402-1523803
Propchange: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/conf/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/conf:r1523402-1523803
Modified:
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/MRClientProtocol.proto
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/MRClientProtocol.proto?rev=1523804&r1=1523803&r2=1523804&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/MRClientProtocol.proto
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/MRClientProtocol.proto
Mon Sep 16 20:59:02 2013
@@ -19,6 +19,7 @@
option java_package = "org.apache.hadoop.yarn.proto";
option java_outer_classname = "MRClientProtocol";
option java_generic_services = true;
+package hadoop.mapreduce;
import "Security.proto";
import "mr_service_protos.proto";
Modified:
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_protos.proto
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_protos.proto?rev=1523804&r1=1523803&r2=1523804&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_protos.proto
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_protos.proto
Mon Sep 16 20:59:02 2013
@@ -20,6 +20,7 @@ option java_package = "org.apache.hadoop
option java_outer_classname = "MRProtos";
option java_generic_services = true;
option java_generate_equals_and_hash = true;
+package hadoop.mapreduce;
import "yarn_protos.proto";
@@ -29,7 +30,7 @@ enum TaskTypeProto {
}
message JobIdProto {
- optional ApplicationIdProto app_id = 1;
+ optional hadoop.yarn.ApplicationIdProto app_id = 1;
optional int32 id = 2;
}
@@ -115,7 +116,7 @@ message TaskAttemptReportProto {
optional string node_manager_host = 12;
optional int32 node_manager_port = 13;
optional int32 node_manager_http_port = 14;
- optional ContainerIdProto container_id = 15;
+ optional hadoop.yarn.ContainerIdProto container_id = 15;
}
enum JobStateProto {
@@ -148,9 +149,9 @@ message JobReportProto {
}
message AMInfoProto {
- optional ApplicationAttemptIdProto application_attempt_id = 1;
+ optional hadoop.yarn.ApplicationAttemptIdProto application_attempt_id = 1;
optional int64 start_time = 2;
- optional ContainerIdProto container_id = 3;
+ optional hadoop.yarn.ContainerIdProto container_id = 3;
optional string node_manager_host = 4;
optional int32 node_manager_port = 5;
optional int32 node_manager_http_port = 6;
Modified:
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_service_protos.proto
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_service_protos.proto?rev=1523804&r1=1523803&r2=1523804&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_service_protos.proto
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_service_protos.proto
Mon Sep 16 20:59:02 2013
@@ -20,6 +20,7 @@ option java_package = "org.apache.hadoop
option java_outer_classname = "MRServiceProtos";
option java_generic_services = true;
option java_generate_equals_and_hash = true;
+package hadoop.mapreduce;
import "Security.proto";
import "mr_protos.proto";
Modified:
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java?rev=1523804&r1=1523803&r2=1523804&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
Mon Sep 16 20:59:02 2013
@@ -24,6 +24,7 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.net.UnknownHostException;
import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
@@ -56,6 +57,7 @@ import org.apache.hadoop.security.Creden
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.util.ReflectionUtils;
import org.codehaus.jackson.JsonParseException;
import org.codehaus.jackson.map.JsonMappingException;
@@ -405,6 +407,19 @@ class JobSubmitter {
// different job.
TokenCache.cleanUpTokenReferral(conf);
+ if (conf.getBoolean(
+ MRJobConfig.JOB_TOKEN_TRACKING_IDS_ENABLED,
+ MRJobConfig.DEFAULT_JOB_TOKEN_TRACKING_IDS_ENABLED)) {
+ // Add HDFS tracking ids
+ ArrayList<String> trackingIds = new ArrayList<String>();
+ for (Token<? extends TokenIdentifier> t :
+ job.getCredentials().getAllTokens()) {
+ trackingIds.add(t.decodeIdentifier().getTrackingId());
+ }
+ conf.setStrings(MRJobConfig.JOB_TOKEN_TRACKING_IDS,
+ trackingIds.toArray(new String[trackingIds.size()]));
+ }
+
// Write job file to submit dir
writeConf(conf, submitJobFile);
Modified:
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java?rev=1523804&r1=1523803&r2=1523804&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
Mon Sep 16 20:59:02 2013
@@ -313,6 +313,13 @@ public interface MRJobConfig {
public static final String MAPREDUCE_JOB_CREDENTIALS_BINARY =
"mapreduce.job.credentials.binary";
+ /* Configs for tracking ids of tokens used by a job */
+ public static final String JOB_TOKEN_TRACKING_IDS_ENABLED =
+ "mapreduce.job.token.tracking.ids.enabled";
+ public static final boolean DEFAULT_JOB_TOKEN_TRACKING_IDS_ENABLED = false;
+ public static final String JOB_TOKEN_TRACKING_IDS =
+ "mapreduce.job.token.tracking.ids";
+
public static final String JOB_SUBMITHOST =
"mapreduce.job.submithostname";
public static final String JOB_SUBMITHOSTADDR =
Modified:
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java?rev=1523804&r1=1523803&r2=1523804&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
Mon Sep 16 20:59:02 2013
@@ -355,8 +355,11 @@ public class MergeManagerImpl<K, V> impl
List<InMemoryMapOutput<K, V>> memory =
new ArrayList<InMemoryMapOutput<K, V>>(inMemoryMergedMapOutputs);
+ inMemoryMergedMapOutputs.clear();
memory.addAll(inMemoryMapOutputs);
+ inMemoryMapOutputs.clear();
List<CompressAwarePath> disk = new
ArrayList<CompressAwarePath>(onDiskMapOutputs);
+ onDiskMapOutputs.clear();
return finalMerge(jobConf, rfs, memory, disk);
}
Modified:
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml?rev=1523804&r1=1523803&r2=1523804&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
Mon Sep 16 20:59:02 2013
@@ -749,6 +749,23 @@
</property>
<property>
+ <name>mapreduce.job.token.tracking.ids.enabled</name>
+ <value>false</value>
+ <description>Whether to write tracking ids of tokens to
+ job-conf. When true, the configuration property
+ "mapreduce.job.token.tracking.ids" is set to the token-tracking-ids of
+ the job</description>
+</property>
+
+<property>
+ <name>mapreduce.job.token.tracking.ids</name>
+ <value></value>
+ <description>When mapreduce.job.token.tracking.ids.enabled is
+ set to true, this is set by the framework to the
+ token-tracking-ids used by the job.</description>
+</property>
+
+<property>
<name>mapreduce.task.merge.progress.records</name>
<value>10000</value>
<description> The number of records to process during merge before
Propchange:
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:r1523402-1523803
Modified:
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMerger.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMerger.java?rev=1523804&r1=1523803&r2=1523804&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMerger.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMerger.java
Mon Sep 16 20:59:02 2013
@@ -82,7 +82,7 @@ public class TestMerger {
}
@Test
- public void testInMemoryMerger() throws IOException {
+ public void testInMemoryMerger() throws Throwable {
JobID jobId = new JobID("a", 0);
TaskAttemptID reduceId = new TaskAttemptID(
new TaskID(jobId, TaskType.REDUCE, 0), 0);
@@ -132,6 +132,11 @@ public class TestMerger {
readOnDiskMapOutput(conf, fs, outPath, keys, values);
Assert.assertEquals(keys, Arrays.asList("apple", "banana", "carrot"));
Assert.assertEquals(values, Arrays.asList("disgusting", "pretty good",
"delicious"));
+
+ mergeManager.close();
+ Assert.assertEquals(0, mergeManager.inMemoryMapOutputs.size());
+ Assert.assertEquals(0, mergeManager.inMemoryMergedMapOutputs.size());
+ Assert.assertEquals(0, mergeManager.onDiskMapOutputs.size());
}
private byte[] writeMapOutput(Configuration conf, Map<String, String>
keysToValues)
Modified:
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/proto/HSAdminRefreshProtocol.proto
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/proto/HSAdminRefreshProtocol.proto?rev=1523804&r1=1523803&r2=1523804&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/proto/HSAdminRefreshProtocol.proto
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/proto/HSAdminRefreshProtocol.proto
Mon Sep 16 20:59:02 2013
@@ -20,6 +20,7 @@ option java_package = "org.apache.hadoop
option java_outer_classname = "HSAdminRefreshProtocolProtos";
option java_generic_services = true;
option java_generate_equals_and_hash = true;
+package hadoop.mapreduce;
/**
* refresh admin acls request.
