Author: todd Date: Fri Feb 10 01:49:08 2012 New Revision: 1242635 URL: http://svn.apache.org/viewvc?rev=1242635&view=rev Log: Merge trunk into HA branch
Added: hadoop/common/branches/HDFS-1623/hadoop-client/ (props changed) - copied from r1242632, hadoop/common/trunk/hadoop-client/ hadoop/common/branches/HDFS-1623/hadoop-client/pom.xml - copied unchanged from r1242632, hadoop/common/trunk/hadoop-client/pom.xml hadoop/common/branches/HDFS-1623/hadoop-minicluster/ (props changed) - copied from r1242632, hadoop/common/trunk/hadoop-minicluster/ hadoop/common/branches/HDFS-1623/hadoop-minicluster/pom.xml - copied unchanged from r1242632, hadoop/common/trunk/hadoop-minicluster/pom.xml Modified: hadoop/common/branches/HDFS-1623/ (props changed) hadoop/common/branches/HDFS-1623/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml hadoop/common/branches/HDFS-1623/hadoop-project/pom.xml hadoop/common/branches/HDFS-1623/hadoop-project/src/site/site.xml hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-archives/ (props changed) hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-distcp/ (props changed) hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-extras/ (props changed) hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-rumen/ (props changed) hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/ZombieJob.java hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-streaming/pom.xml hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingBadRecords.java hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestUlimit.java hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-tools-dist/ (props changed) hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-tools-dist/pom.xml hadoop/common/branches/HDFS-1623/pom.xml Propchange: hadoop/common/branches/HDFS-1623/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Fri Feb 10 01:49:08 2012 @@ -1 +1 @@ -/hadoop/common/trunk:1152502-1237154 +/hadoop/common/trunk:1152502-1242632 Modified: hadoop/common/branches/HDFS-1623/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml (original) +++ hadoop/common/branches/HDFS-1623/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml Fri Feb 10 01:49:08 2012 @@ -72,6 +72,14 @@ <fileMode>0755</fileMode> </fileSet> <fileSet> + <directory>bin</directory> + <outputDirectory>sbin</outputDirectory> + <includes> + <include>mr-jobhistory-daemon.sh</include> + </includes> + <fileMode>0755</fileMode> + </fileSet> + <fileSet> <directory>hadoop-yarn/conf</directory> <outputDirectory>etc/hadoop</outputDirectory> <includes> Propchange: hadoop/common/branches/HDFS-1623/hadoop-client/ ------------------------------------------------------------------------------ --- svn:ignore (added) +++ svn:ignore Fri Feb 10 01:49:08 2012 @@ -0,0 +1 @@ +target Propchange: hadoop/common/branches/HDFS-1623/hadoop-minicluster/ ------------------------------------------------------------------------------ --- svn:ignore (added) +++ svn:ignore Fri Feb 10 01:49:08 2012 @@ -0,0 +1 @@ +target Modified: hadoop/common/branches/HDFS-1623/hadoop-project/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-project/pom.xml?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-project/pom.xml (original) +++ hadoop/common/branches/HDFS-1623/hadoop-project/pom.xml Fri Feb 10 01:49:08 2012 @@ -524,7 +524,7 @@ <dependency> <groupId>org.codehaus.jackson</groupId> <artifactId>jackson-mapper-asl</artifactId> - <version>1.7.1</version> + <version>1.8.8</version> </dependency> <dependency> <groupId>org.aspectj</groupId> @@ -667,7 +667,7 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-javadoc-plugin</artifactId> - <version>2.7</version> + <version>2.8.1</version> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> @@ -740,25 +740,6 @@ <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-antrun-plugin</artifactId> - <executions> - <execution> - <id>create-testdirs</id> - <phase>validate</phase> - <goals> - <goal>run</goal> - </goals> - <configuration> - <target> - <mkdir dir="${test.build.dir}"/> - <mkdir dir="${test.build.data}"/> - </target> - </configuration> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-plugin</artifactId> <configuration> <forkMode>always</forkMode> @@ -906,5 +887,53 @@ </plugins> </build> </profile> + <profile> + <id>dist</id> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-javadoc-plugin</artifactId> + <executions> + <execution> + <!-- build javadoc jars per jar for publishing to maven --> + <id>module-javadocs</id> + <phase>package</phase> + <goals> + <goal>jar</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-source-plugin</artifactId> + <executions> + <execution> + <!-- builds source jars and attaches them to the project for publishing --> + <id>hadoop-java-sources</id> + <phase>package</phase> + <goals> + <goal>jar-no-fork</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-enforcer-plugin</artifactId> + <executions> + <execution> + <id>dist-enforce</id> + <phase>package</phase> + <goals> + <goal>enforce</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> </profiles> </project> Modified: hadoop/common/branches/HDFS-1623/hadoop-project/src/site/site.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-project/src/site/site.xml?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-project/src/site/site.xml (original) +++ hadoop/common/branches/HDFS-1623/hadoop-project/src/site/site.xml Fri Feb 10 01:49:08 2012 @@ -91,7 +91,7 @@ <menu name="Configuration" inherit="top"> <item name="core-default.xml" href="hadoop-project-dist/hadoop-common/core-default.xml"/> <item name="hdfs-default.xml" href="hadoop-project-dist/hadoop-hdfs/hdfs-default.xml"/> - <item name="yarn-default.xml" href="hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/yarn-default.xml"/> + <item name="yarn-default.xml" href="hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml"/> <item name="mapred-default.xml" href="hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml"/> <item name="Deprecated Properties" href="hadoop-project-dist/hadoop-common/DeprecatedProperties.html"/> </menu> Propchange: hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-archives/ ------------------------------------------------------------------------------ --- svn:ignore (added) +++ svn:ignore Fri Feb 10 01:49:08 2012 @@ -0,0 +1 @@ +target Propchange: hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-distcp/ ------------------------------------------------------------------------------ --- svn:ignore (added) +++ svn:ignore Fri Feb 10 01:49:08 2012 @@ -0,0 +1 @@ +target Propchange: hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-extras/ ------------------------------------------------------------------------------ --- svn:ignore (added) +++ svn:ignore Fri Feb 10 01:49:08 2012 @@ -0,0 +1 @@ +target Propchange: hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-rumen/ ------------------------------------------------------------------------------ --- svn:ignore (added) +++ svn:ignore Fri Feb 10 01:49:08 2012 @@ -0,0 +1 @@ +target Modified: hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/ZombieJob.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/ZombieJob.java?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/ZombieJob.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/ZombieJob.java Fri Feb 10 01:49:08 2012 @@ -229,7 +229,7 @@ public class ZombieJob implements JobSto @Override public String getName() { JobName jobName = job.getJobName(); - if (jobName == null) { + if (jobName == null || jobName.getValue() == null) { return "(name unknown)"; } else { return jobName.getValue(); Modified: hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-streaming/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-streaming/pom.xml?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-streaming/pom.xml (original) +++ hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-streaming/pom.xml Fri Feb 10 01:49:08 2012 @@ -29,7 +29,7 @@ <properties> <hadoop.log.dir>${project.build.directory}/log</hadoop.log.dir> - <test.exclude.pattern>%regex[.*(TestStreamingBadRecords|TestStreamingStatus|TestUlimit).*]</test.exclude.pattern> + <test.exclude.pattern>%regex[.*(TestStreamingStatus).*]</test.exclude.pattern> </properties> <dependencies> Modified: hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingBadRecords.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingBadRecords.java?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingBadRecords.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingBadRecords.java Fri Feb 10 01:49:08 2012 @@ -154,6 +154,10 @@ public class TestStreamingBadRecords ext } } + /* + * Disable test as skipping bad records not supported in 0.23 + */ + /* public void testSkip() throws Exception { JobConf clusterConf = createJobConf(); createInput(); @@ -195,7 +199,12 @@ public class TestStreamingBadRecords ext //validate that there is no skip directory as it has been set to "none" assertTrue(SkipBadRecords.getSkipOutputPath(job.jobConf_)==null); } + */ + /* + * Disable test as skipping bad records not supported in 0.23 + */ + /* public void testNarrowDown() throws Exception { createInput(); JobConf clusterConf = createJobConf(); @@ -231,6 +240,11 @@ public class TestStreamingBadRecords ext validateOutput(job.running_, true); assertTrue(SkipBadRecords.getSkipOutputPath(job.jobConf_)!=null); } + */ + + public void testNoOp() { + // Added to avoid warnings when running this disabled test + } static class App{ boolean isReducer; Modified: hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestUlimit.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestUlimit.java?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestUlimit.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestUlimit.java Fri Feb 10 01:49:08 2012 @@ -31,6 +31,7 @@ import org.apache.hadoop.mapreduce.MapRe import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; import org.apache.hadoop.util.StringUtils; +import org.junit.Ignore; import org.junit.Test; import static org.junit.Assert.*; @@ -52,7 +53,6 @@ public class TestUlimit { private static String SET_MEMORY_LIMIT = "786432"; // 768MB String[] genArgs(String memLimit) { - String strJobtracker = JTConfig.JT_IPC_ADDRESS + "=localhost:" + mr.createJobConf().get(JTConfig.JT_IPC_ADDRESS); String strNamenode = "fs.default.name=" + mr.createJobConf().get("fs.default.name"); return new String[] { "-input", inputPath.toString(), @@ -63,7 +63,6 @@ public class TestUlimit { "-jobconf", MRJobConfig.NUM_MAPS + "=1", "-jobconf", JobConf.MAPRED_MAP_TASK_ULIMIT + "=" + memLimit, "-jobconf", strNamenode, - "-jobconf", strJobtracker, "-jobconf", "stream.tmpdir=" + System.getProperty("test.build.data","/tmp"), "-jobconf", "mapred.jar=" + TestStreaming.STREAMING_JAR, @@ -79,6 +78,7 @@ public class TestUlimit { * is expected to be a failure. */ @Test + @Ignore public void testCommandLine() { if (UtilTest.isCygwin()) { return; Propchange: hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-tools-dist/ ------------------------------------------------------------------------------ --- svn:ignore (added) +++ svn:ignore Fri Feb 10 01:49:08 2012 @@ -0,0 +1 @@ +target Modified: hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-tools-dist/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-tools-dist/pom.xml?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-tools-dist/pom.xml (original) +++ hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-tools-dist/pom.xml Fri Feb 10 01:49:08 2012 @@ -29,7 +29,6 @@ <properties> <hadoop.component>tools</hadoop.component> - <is.hadoop.component>false</is.hadoop.component> </properties> <dependencies> Modified: hadoop/common/branches/HDFS-1623/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/pom.xml?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/pom.xml (original) +++ hadoop/common/branches/HDFS-1623/pom.xml Fri Feb 10 01:49:08 2012 @@ -79,6 +79,8 @@ <module>hadoop-mapreduce-project</module> <module>hadoop-tools</module> <module>hadoop-dist</module> + <module>hadoop-client</module> + <module>hadoop-minicluster</module> </modules> <build> @@ -88,6 +90,16 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-enforcer-plugin</artifactId> <version>1.0</version> + <configuration> + <rules> + <requireMavenVersion> + <version>[3.0.2,)</version> + </requireMavenVersion> + <requireJavaVersion> + <version>1.6</version> + </requireJavaVersion> + </rules> + </configuration> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> @@ -108,6 +120,120 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-site-plugin</artifactId> <version>3.0</version> + <dependencies> + <dependency><!-- add support for ssh/scp --> + <groupId>org.apache.maven.wagon</groupId> + <artifactId>wagon-ssh</artifactId> + <version>1.0</version> + </dependency> + </dependencies> + </plugin> + <!--This plugin's configuration is used to store Eclipse m2e settings only. + It has no influence on the Maven build itself.--> + <plugin> + <groupId>org.eclipse.m2e</groupId> + <artifactId>lifecycle-mapping</artifactId> + <version>1.0.0</version> + <configuration> + <lifecycleMappingMetadata> + <pluginExecutions> + <pluginExecution> + <pluginExecutionFilter> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <versionRange>[1.6,)</versionRange> + <goals> + <goal>run</goal> + </goals> + </pluginExecutionFilter> + <action> + <ignore></ignore> + </action> + </pluginExecution> + <pluginExecution> + <pluginExecutionFilter> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-resources-plugin</artifactId> + <versionRange>[2.2,)</versionRange> + <goals> + <goal>testResources</goal> + <goal>resources</goal> + </goals> + </pluginExecutionFilter> + <action> + <ignore></ignore> + </action> + </pluginExecution> + <pluginExecution> + <pluginExecutionFilter> + <groupId>org.apache.avro</groupId> + <artifactId>avro-maven-plugin</artifactId> + <versionRange>[1.5.3,)</versionRange> + <goals> + <goal>schema</goal> + <goal>protocol</goal> + </goals> + </pluginExecutionFilter> + <action> + <ignore></ignore> + </action> + </pluginExecution> + <pluginExecution> + <pluginExecutionFilter> + <groupId>org.codehaus.mojo.jspc</groupId> + <artifactId>jspc-maven-plugin</artifactId> + <versionRange>[2.0-alpha-3,)</versionRange> + <goals> + <goal>compile</goal> + </goals> + </pluginExecutionFilter> + <action> + <ignore></ignore> + </action> + </pluginExecution> + <pluginExecution> + <pluginExecutionFilter> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <versionRange>[2.1,)</versionRange> + <goals> + <goal>copy-dependencies</goal> + <goal>build-classpath</goal> + </goals> + </pluginExecutionFilter> + <action> + <ignore></ignore> + </action> + </pluginExecution> + <pluginExecution> + <pluginExecutionFilter> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <versionRange>[1.2,)</versionRange> + <goals> + <goal>exec</goal> + </goals> + </pluginExecutionFilter> + <action> + <ignore></ignore> + </action> + </pluginExecution> + <pluginExecution> + <pluginExecutionFilter> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <versionRange>[2.3.1,)</versionRange> + <goals> + <goal>test-jar</goal> + </goals> + </pluginExecutionFilter> + <action> + <ignore></ignore> + </action> + </pluginExecution> + </pluginExecutions> + </lifecycleMappingMetadata> + </configuration> </plugin> </plugins> </pluginManagement> @@ -117,16 +243,6 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-enforcer-plugin</artifactId> <inherited>false</inherited> - <configuration> - <rules> - <requireMavenVersion> - <version>[3.0.2,)</version> - </requireMavenVersion> - <requireJavaVersion> - <version>1.6</version> - </requireJavaVersion> - </rules> - </configuration> <executions> <execution> <id>clean</id> @@ -185,7 +301,8 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-javadoc-plugin</artifactId> - <version>2.8</version> + <version>2.8.1</version> + <inherited>false</inherited> <reportSets> <reportSet> <id>aggregate</id> @@ -254,6 +371,7 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-assembly-plugin</artifactId> + <inherited>false</inherited> <executions> <execution> <id>src-dist</id> @@ -278,6 +396,7 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-antrun-plugin</artifactId> + <inherited>false</inherited> <executions> <execution> <id>src-dist-msg</id> @@ -307,54 +426,20 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-javadoc-plugin</artifactId> + <inherited>false</inherited> <executions> <execution> - <!-- build javadoc jars per jar for publishing to maven --> - <id>module-javadocs</id> - <phase>package</phase> - <goals> - <goal>jar</goal> - </goals> - </execution> - <execution> <!-- build aggregate javadoc in parent only --> <id>default-cli</id> <goals> <goal>aggregate</goal> </goals> - <inherited>false</inherited> <configuration> <overview>hadoop-common-project/hadoop-common/src/main/java/overview.html</overview> </configuration> </execution> </executions> </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-source-plugin</artifactId> - <executions> - <execution> - <!-- builds source jars and attaches them to the project for publishing --> - <id>hadoop-java-sources</id> - <phase>package</phase> - <goals> - <goal>jar-no-fork</goal> - </goals> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-enforcer-plugin</artifactId> - <executions> - <execution> - <phase>package</phase> - <goals> - <goal>enforce</goal> - </goals> - </execution> - </executions> - </plugin> </plugins> </build> </profile>