Author: tucu Date: Mon Apr 1 21:42:12 2013 New Revision: 1463307 URL: http://svn.apache.org/r1463307 Log: MAPREDUCE-5113. Streaming input/output types are ignored with java mapper/reducer. (sandyr via tucu)
Modified: hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingOutputKeyValueTypes.java hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java Modified: hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java?rev=1463307&r1=1463306&r2=1463307&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java (original) +++ hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java Mon Apr 1 21:42:12 2013 @@ -875,7 +875,7 @@ public class StreamJob implements Tool { IdentifierResolver.TEXT_ID)); jobConf_.setClass("stream.map.output.reader.class", idResolver.getOutputReaderClass(), OutputReader.class); - if (isMapperACommand) { + if (isMapperACommand || jobConf_.get("stream.map.output") != null) { // if mapper is a command, then map output key/value classes come from the // idResolver jobConf_.setMapOutputKeyClass(idResolver.getOutputKeyClass()); @@ -891,7 +891,7 @@ public class StreamJob implements Tool { IdentifierResolver.TEXT_ID)); jobConf_.setClass("stream.reduce.output.reader.class", idResolver.getOutputReaderClass(), OutputReader.class); - if (isReducerACommand) { + if (isReducerACommand || jobConf_.get("stream.reduce.output") != null) { // if reducer is a command, then output key/value classes come from the // idResolver jobConf_.setOutputKeyClass(idResolver.getOutputKeyClass()); Modified: hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingOutputKeyValueTypes.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingOutputKeyValueTypes.java?rev=1463307&r1=1463306&r2=1463307&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingOutputKeyValueTypes.java (original) +++ hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingOutputKeyValueTypes.java Mon Apr 1 21:42:12 2013 @@ -24,6 +24,7 @@ import org.apache.hadoop.mapred.MapReduc import org.apache.hadoop.mapred.OutputCollector; import org.apache.hadoop.mapred.Reducer; import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.mapred.TextInputFormat; import org.apache.hadoop.mapreduce.MRJobConfig; import org.junit.Before; import org.junit.Test; @@ -177,6 +178,17 @@ public class TestStreamingOutputKeyValue args.add("0"); super.testCommandLine(); } + + @Test + public void testDefaultToIdentityReducer() throws Exception { + args.add("-mapper");args.add(map); + args.add("-jobconf"); + args.add("mapreduce.task.files.preserve.failedtasks=true"); + args.add("-jobconf"); + args.add("stream.tmpdir="+System.getProperty("test.build.data","/tmp")); + args.add("-inputformat");args.add(TextInputFormat.class.getName()); + super.testCommandLine(); + } @Override @Test Modified: hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java?rev=1463307&r1=1463306&r2=1463307&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java (original) +++ hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java Mon Apr 1 21:42:12 2013 @@ -43,7 +43,6 @@ public class TrApp // Note the dots translated to underscore: // property names have been escaped in PipeMapRed.safeEnvVarName() expectDefined("mapreduce_cluster_local_dir"); - expect("mapred_output_format_class", "org.apache.hadoop.mapred.TextOutputFormat"); expect("mapreduce_map_output_key_class", "org.apache.hadoop.io.Text"); expect("mapreduce_map_output_value_class", "org.apache.hadoop.io.Text");