Author: tgraves
Date: Tue Jul 31 13:44:50 2012
New Revision: 1367544
URL: http://svn.apache.org/viewvc?rev=1367544&view=rev
Log:
merge -r 1367539:1367540 from branch-2. FIXES: MAPREDUCE-4375
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfEntryInfo.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfInfo.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/MapredAppMasterRest.apt.vm
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1367544&r1=1367543&r2=1367544&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Tue
Jul 31 13:44:50 2012
@@ -56,6 +56,9 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-2220. Fix new API FileOutputFormat-related typos in
mapred-default.xml (Rui Kubo via harsh)
+ MAPREDUCE-4375. Show Configuration Tracability in MR UI (bobby
+ via tgraves)
+
OPTIMIZATIONS
MAPREDUCE-3850. Avoid redundant calls for tokens in TokenCache (Daryn
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1367544&r1=1367543&r2=1367544&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
Tue Jul 31 13:44:50 2012
@@ -1481,7 +1481,7 @@ public class JobImpl implements org.apac
Path confPath = getConfFile();
FileContext fc = FileContext.getFileContext(confPath.toUri(), conf);
Configuration jobConf = new Configuration(false);
- jobConf.addResource(fc.open(confPath));
+ jobConf.addResource(fc.open(confPath), confPath.toString());
return jobConf;
}
}
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java?rev=1367544&r1=1367543&r2=1367544&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java
Tue Jul 31 13:44:50 2012
@@ -78,14 +78,29 @@ public class ConfBlock extends HtmlBlock
tr().
th(_TH, "key").
th(_TH, "value").
+ th(_TH, "source chain").
_().
_().
tbody();
for (ConfEntryInfo entry : info.getProperties()) {
+ StringBuffer buffer = new StringBuffer();
+ String[] sources = entry.getSource();
+ //Skip the last entry, because it is always the same HDFS file, and
+ // output them in reverse order so most recent is output first
+ boolean first = true;
+ for(int i = (sources.length - 2); i >= 0; i--) {
+ if(!first) {
+ // \u2B05 is an arrow <--
+ buffer.append(" \u2B05 ");
+ }
+ first = false;
+ buffer.append(sources[i]);
+ }
tbody.
tr().
td(entry.getName()).
td(entry.getValue()).
+ td(buffer.toString()).
_();
}
tbody._().
@@ -93,6 +108,7 @@ public class ConfBlock extends HtmlBlock
tr().
th().input("search_init").$type(InputType.text).$name("key").$value("key")._()._().
th().input("search_init").$type(InputType.text).$name("value").$value("value")._()._().
+ th().input("search_init").$type(InputType.text).$name("source
chain").$value("source chain")._()._().
_().
_().
_();
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfEntryInfo.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfEntryInfo.java?rev=1367544&r1=1367543&r2=1367544&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfEntryInfo.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfEntryInfo.java
Tue Jul 31 13:44:50 2012
@@ -27,13 +27,19 @@ public class ConfEntryInfo {
protected String name;
protected String value;
+ protected String[] source;
public ConfEntryInfo() {
}
public ConfEntryInfo(String key, String value) {
+ this(key, value, null);
+ }
+
+ public ConfEntryInfo(String key, String value, String[] source) {
this.name = key;
this.value = value;
+ this.source = source;
}
public String getName() {
@@ -43,4 +49,8 @@ public class ConfEntryInfo {
public String getValue() {
return this.value;
}
+
+ public String[] getSource() {
+ return source;
+ }
}
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfInfo.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfInfo.java?rev=1367544&r1=1367543&r2=1367544&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfInfo.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfInfo.java
Tue Jul 31 13:44:50 2012
@@ -46,7 +46,8 @@ public class ConfInfo {
Configuration jobConf = job.loadConfFile();
this.path = job.getConfFile().toString();
for (Map.Entry<String, String> entry : jobConf) {
- this.property.add(new ConfEntryInfo(entry.getKey(), entry.getValue()));
+ this.property.add(new ConfEntryInfo(entry.getKey(), entry.getValue(),
+ jobConf.getPropertySources(entry.getKey())));
}
}
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java?rev=1367544&r1=1367543&r2=1367544&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java
Tue Jul 31 13:44:50 2012
@@ -597,7 +597,7 @@ public class MockJobs extends MockApps {
public Configuration loadConfFile() throws IOException {
FileContext fc = FileContext.getFileContext(configFile.toUri(), conf);
Configuration jobConf = new Configuration(false);
- jobConf.addResource(fc.open(configFile));
+ jobConf.addResource(fc.open(configFile), configFile.toString());
return jobConf;
}
};
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java?rev=1367544&r1=1367543&r2=1367544&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
Tue Jul 31 13:44:50 2012
@@ -336,7 +336,7 @@ public class HistoryFileManager extends
public synchronized Configuration loadConfFile() throws IOException {
FileContext fc = FileContext.getFileContext(confFile.toUri(), conf);
Configuration jobConf = new Configuration(false);
- jobConf.addResource(fc.open(confFile));
+ jobConf.addResource(fc.open(confFile), confFile.toString());
return jobConf;
}
}
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm?rev=1367544&r1=1367543&r2=1367544&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm
Tue Jul 31 13:44:50 2012
@@ -1261,6 +1261,9 @@ History Server REST API's.
*---------------+--------------+-------------------------------+
| value | string | The value of the configuration property |
*---------------+--------------+-------------------------------+
+| source | string | The location this configuration object came from. If there
is more then one of these it shows the history with the latest source at the
end of the list. |
+*---------------+--------------+-------------------------------+
+
*** Response Examples
@@ -1293,14 +1296,17 @@ History Server REST API's.
{
"value" : "/home/hadoop/hdfs/data",
"name" : "dfs.datanode.data.dir"
+ "source" : ["hdfs-site.xml", "job.xml"]
},
{
"value" :
"org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer",
"name" : "hadoop.http.filter.initializers"
+ "source" : ["programatically", "job.xml"]
},
{
"value" : "/home/hadoop/tmp",
"name" : "mapreduce.cluster.temp.dir"
+ "source" : ["mapred-site.xml"]
},
...
]
@@ -1335,14 +1341,19 @@ History Server REST API's.
<property>
<name>dfs.datanode.data.dir</name>
<value>/home/hadoop/hdfs/data</value>
+ <source>hdfs-site.xml</source>
+ <source>job.xml</source>
</property>
<property>
<name>hadoop.http.filter.initializers</name>
<value>org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer</value>
+ <source>programatically</source>
+ <source>job.xml</source>
</property>
<property>
<name>mapreduce.cluster.temp.dir</name>
<value>/home/hadoop/tmp</value>
+ <source>mapred-site.xml</source>
</property>
...
</conf>
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/MapredAppMasterRest.apt.vm
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/MapredAppMasterRest.apt.vm?rev=1367544&r1=1367543&r2=1367544&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/MapredAppMasterRest.apt.vm
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/MapredAppMasterRest.apt.vm
Tue Jul 31 13:44:50 2012
@@ -1296,6 +1296,8 @@ MapReduce Application Master REST API's.
*---------------+--------------+-------------------------------+
| value | string | The value of the configuration property |
*---------------+--------------+-------------------------------+
+| source | string | The location this configuration object came from. If there
is more then one of these it shows the history with the latest source at the
end of the list. |
+*---------------+--------------+-------------------------------+
** Response Examples
@@ -1327,15 +1329,18 @@ MapReduce Application Master REST API's.
"property" : [
{
"value" : "/home/hadoop/hdfs/data",
- "name" : "dfs.datanode.data.dir"
+ "name" : "dfs.datanode.data.dir",
+ "source" : ["hdfs-site.xml", "job.xml"]
},
{
"value" :
"org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer",
"name" : "hadoop.http.filter.initializers"
+ "source" : ["programatically", "job.xml"]
},
{
"value" : "/home/hadoop/tmp",
"name" : "mapreduce.cluster.temp.dir"
+ "source" : ["mapred-site.xml"]
},
...
]
@@ -1370,14 +1375,19 @@ MapReduce Application Master REST API's.
<property>
<name>dfs.datanode.data.dir</name>
<value>/home/hadoop/hdfs/data</value>
+ <source>hdfs-site.xml</source>
+ <source>job.xml</source>
</property>
<property>
<name>hadoop.http.filter.initializers</name>
<value>org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer</value>
+ <source>programatically</source>
+ <source>job.xml</source>
</property>
<property>
<name>mapreduce.cluster.temp.dir</name>
<value>/home/hadoop/tmp</value>
+ <source>mapred-site.xml</source>
</property>
...
</conf>
