git commit: MAPREDUCE-6022. map_input_file is missing from streaming job environment. Contributed by Jason Lowe.
Repository: hadoop
Updated Branches:
refs/heads/trunk 782971ae7 - b05604811
MAPREDUCE-6022. map_input_file is missing from streaming job
environment. Contributed by Jason Lowe.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b0560481
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b0560481
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b0560481
Branch: refs/heads/trunk
Commit: b056048114bf4701ef9dd22486db937cb589e81b
Parents: 782971a
Author: Kihwal Lee kih...@apache.org
Authored: Wed Oct 29 12:28:32 2014 -0500
Committer: Kihwal Lee kih...@apache.org
Committed: Wed Oct 29 12:29:07 2014 -0500
--
.../org/apache/hadoop/conf/Configuration.java | 26
hadoop-mapreduce-project/CHANGES.txt| 3 +++
.../org/apache/hadoop/streaming/PipeMapRed.java | 4 ++-
.../java/org/apache/hadoop/streaming/TrApp.java | 3 +++
4 files changed, 35 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0560481/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index 98fa4f6..a3fae19 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -559,6 +559,32 @@ public class Configuration implements
IterableMap.EntryString,String,
}
/**
+ * Sets all deprecated properties that are not currently set but have a
+ * corresponding new property that is set. Useful for iterating the
+ * properties when all deprecated properties for currently set properties
+ * need to be present.
+ */
+ public void setDeprecatedProperties() {
+DeprecationContext deprecations = deprecationContext.get();
+Properties props = getProps();
+Properties overlay = getOverlay();
+for (Map.EntryString, DeprecatedKeyInfo entry :
+deprecations.getDeprecatedKeyMap().entrySet()) {
+ String depKey = entry.getKey();
+ if (!overlay.contains(depKey)) {
+for (String newKey : entry.getValue().newKeys) {
+ String val = overlay.getProperty(newKey);
+ if (val != null) {
+props.setProperty(depKey, val);
+overlay.setProperty(depKey, val);
+break;
+ }
+}
+ }
+}
+ }
+
+ /**
* Checks for the presence of the property codename/code in the
* deprecation map. Returns the first of the list of new keys if present
* in the deprecation map or the codename/code itself. If the property
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0560481/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index c0edb37..2c06aa3 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -450,6 +450,9 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-6142. Fixed test failures in TestJobHistoryEventHandler and
TestMRTimelineEventHandling. (Zhijie Shen via vinodkv)
+MAPREDUCE-6022. map_input_file is missing from streaming job environment.
+(jlowe via kihwal)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0560481/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/PipeMapRed.java
--
diff --git
a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/PipeMapRed.java
b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/PipeMapRed.java
index c8d099e..f47e756 100644
---
a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/PipeMapRed.java
+++
b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/PipeMapRed.java
@@ -235,7 +235,9 @@ public abstract class PipeMapRed {
}
}
- void addJobConfToEnvironment(JobConf conf, Properties env) {
+ void addJobConfToEnvironment(JobConf jobconf, Properties env) {
+JobConf conf = new JobConf(jobconf);
+conf.setDeprecatedProperties();
Iterator it = conf.iterator();
while (it.hasNext()) {
Map.Entry en = (Map.Entry) it.next();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0560481/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java
git commit: MAPREDUCE-6022. map_input_file is missing from streaming job environment. Contributed by Jason Lowe. (cherry picked from commit b056048114bf4701ef9dd22486db937cb589e81b)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 e88832dfb - 45b6dea95
MAPREDUCE-6022. map_input_file is missing from streaming job
environment. Contributed by Jason Lowe.
(cherry picked from commit b056048114bf4701ef9dd22486db937cb589e81b)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/45b6dea9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/45b6dea9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/45b6dea9
Branch: refs/heads/branch-2
Commit: 45b6dea95f5397282e2611f3dc6f68df0527529c
Parents: e88832d
Author: Kihwal Lee kih...@apache.org
Authored: Wed Oct 29 12:30:08 2014 -0500
Committer: Kihwal Lee kih...@apache.org
Committed: Wed Oct 29 12:30:08 2014 -0500
--
.../org/apache/hadoop/conf/Configuration.java | 26
hadoop-mapreduce-project/CHANGES.txt| 3 +++
.../org/apache/hadoop/streaming/PipeMapRed.java | 4 ++-
.../java/org/apache/hadoop/streaming/TrApp.java | 3 +++
4 files changed, 35 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/45b6dea9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index 62cc78d..1276865 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -559,6 +559,32 @@ public class Configuration implements
IterableMap.EntryString,String,
}
/**
+ * Sets all deprecated properties that are not currently set but have a
+ * corresponding new property that is set. Useful for iterating the
+ * properties when all deprecated properties for currently set properties
+ * need to be present.
+ */
+ public void setDeprecatedProperties() {
+DeprecationContext deprecations = deprecationContext.get();
+Properties props = getProps();
+Properties overlay = getOverlay();
+for (Map.EntryString, DeprecatedKeyInfo entry :
+deprecations.getDeprecatedKeyMap().entrySet()) {
+ String depKey = entry.getKey();
+ if (!overlay.contains(depKey)) {
+for (String newKey : entry.getValue().newKeys) {
+ String val = overlay.getProperty(newKey);
+ if (val != null) {
+props.setProperty(depKey, val);
+overlay.setProperty(depKey, val);
+break;
+ }
+}
+ }
+}
+ }
+
+ /**
* Checks for the presence of the property codename/code in the
* deprecation map. Returns the first of the list of new keys if present
* in the deprecation map or the codename/code itself. If the property
http://git-wip-us.apache.org/repos/asf/hadoop/blob/45b6dea9/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index d60e3d5..6e245ec 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -231,6 +231,9 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-6142. Fixed test failures in TestJobHistoryEventHandler and
TestMRTimelineEventHandling. (Zhijie Shen via vinodkv)
+MAPREDUCE-6022. map_input_file is missing from streaming job environment.
+(jlowe via kihwal)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/45b6dea9/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/PipeMapRed.java
--
diff --git
a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/PipeMapRed.java
b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/PipeMapRed.java
index c8d099e..f47e756 100644
---
a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/PipeMapRed.java
+++
b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/PipeMapRed.java
@@ -235,7 +235,9 @@ public abstract class PipeMapRed {
}
}
- void addJobConfToEnvironment(JobConf conf, Properties env) {
+ void addJobConfToEnvironment(JobConf jobconf, Properties env) {
+JobConf conf = new JobConf(jobconf);
+conf.setDeprecatedProperties();
Iterator it = conf.iterator();
while (it.hasNext()) {
Map.Entry en = (Map.Entry) it.next();
git commit: MAPREDUCE-6022. map_input_file is missing from streaming job environment. Contributed by Jason Lowe. (cherry picked from commit b056048114bf4701ef9dd22486db937cb589e81b)
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 6add7b19f - afbb27982
MAPREDUCE-6022. map_input_file is missing from streaming job
environment. Contributed by Jason Lowe.
(cherry picked from commit b056048114bf4701ef9dd22486db937cb589e81b)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/afbb2798
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/afbb2798
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/afbb2798
Branch: refs/heads/branch-2.6
Commit: afbb2798253ed7dae38ac3ce1ab407466637c829
Parents: 6add7b1
Author: Kihwal Lee kih...@apache.org
Authored: Wed Oct 29 12:30:51 2014 -0500
Committer: Kihwal Lee kih...@apache.org
Committed: Wed Oct 29 12:30:51 2014 -0500
--
.../org/apache/hadoop/conf/Configuration.java | 26
hadoop-mapreduce-project/CHANGES.txt| 3 +++
.../org/apache/hadoop/streaming/PipeMapRed.java | 4 ++-
.../java/org/apache/hadoop/streaming/TrApp.java | 3 +++
4 files changed, 35 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/afbb2798/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index 62cc78d..1276865 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -559,6 +559,32 @@ public class Configuration implements
IterableMap.EntryString,String,
}
/**
+ * Sets all deprecated properties that are not currently set but have a
+ * corresponding new property that is set. Useful for iterating the
+ * properties when all deprecated properties for currently set properties
+ * need to be present.
+ */
+ public void setDeprecatedProperties() {
+DeprecationContext deprecations = deprecationContext.get();
+Properties props = getProps();
+Properties overlay = getOverlay();
+for (Map.EntryString, DeprecatedKeyInfo entry :
+deprecations.getDeprecatedKeyMap().entrySet()) {
+ String depKey = entry.getKey();
+ if (!overlay.contains(depKey)) {
+for (String newKey : entry.getValue().newKeys) {
+ String val = overlay.getProperty(newKey);
+ if (val != null) {
+props.setProperty(depKey, val);
+overlay.setProperty(depKey, val);
+break;
+ }
+}
+ }
+}
+ }
+
+ /**
* Checks for the presence of the property codename/code in the
* deprecation map. Returns the first of the list of new keys if present
* in the deprecation map or the codename/code itself. If the property
http://git-wip-us.apache.org/repos/asf/hadoop/blob/afbb2798/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 5d7dc8c..ab9d9c3 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -216,6 +216,9 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-6142. Fixed test failures in TestJobHistoryEventHandler and
TestMRTimelineEventHandling. (Zhijie Shen via vinodkv)
+MAPREDUCE-6022. map_input_file is missing from streaming job environment.
+(jlowe via kihwal)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/afbb2798/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/PipeMapRed.java
--
diff --git
a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/PipeMapRed.java
b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/PipeMapRed.java
index c8d099e..f47e756 100644
---
a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/PipeMapRed.java
+++
b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/PipeMapRed.java
@@ -235,7 +235,9 @@ public abstract class PipeMapRed {
}
}
- void addJobConfToEnvironment(JobConf conf, Properties env) {
+ void addJobConfToEnvironment(JobConf jobconf, Properties env) {
+JobConf conf = new JobConf(jobconf);
+conf.setDeprecatedProperties();
Iterator it = conf.iterator();
while (it.hasNext()) {
Map.Entry en = (Map.Entry) it.next();
