[Hadoop Wiki] Update of "HadoopMeetupAtApacheCon" by KonstantinShvachko
Dear Wiki user, You have subscribed to a wiki page or wiki category on "Hadoop Wiki" for change notification. The "HadoopMeetupAtApacheCon" page has been changed by KonstantinShvachko: http://wiki.apache.org/hadoop/HadoopMeetupAtApacheCon?action=diff&rev1=5&rev2=6 (Please add your name here if you plan to attend.) 1. Owen O'Malley - 2. Milind Bhandarkar + 1. Milind Bhandarkar - 3. Allen Wittenauer + 1. Allen Wittenauer + 1. Konstantin Shvachko
svn commit: r1182018 - in /hadoop/common/branches/branch-0.22/common: CHANGES.txt src/test/core/org/apache/hadoop/cli/CLITestHelper.java src/test/core/org/apache/hadoop/cli/util/CommandExecutor.java
Author: cos Date: Tue Oct 11 18:43:13 2011 New Revision: 1182018 URL: http://svn.apache.org/viewvc?rev=1182018&view=rev Log: HADOOP-7730. Allow TestCLI to be run against a cluster. Contributed by Tom White, Konstantin Boudnik. Modified: hadoop/common/branches/branch-0.22/common/CHANGES.txt hadoop/common/branches/branch-0.22/common/src/test/core/org/apache/hadoop/cli/CLITestHelper.java hadoop/common/branches/branch-0.22/common/src/test/core/org/apache/hadoop/cli/util/CommandExecutor.java Modified: hadoop/common/branches/branch-0.22/common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/common/CHANGES.txt?rev=1182018&r1=1182017&r2=1182018&view=diff == --- hadoop/common/branches/branch-0.22/common/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.22/common/CHANGES.txt Tue Oct 11 18:43:13 2011 @@ -549,6 +549,8 @@ Release 0.22.0 - Unreleased HADOOP-7697. Remove ivy dependency on different version of slf4j in avro. (shv) +HADOOP-7730. Allow TestCLI to be run against a cluster (cos) + Release 0.21.1 - Unreleased IMPROVEMENTS Modified: hadoop/common/branches/branch-0.22/common/src/test/core/org/apache/hadoop/cli/CLITestHelper.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/common/src/test/core/org/apache/hadoop/cli/CLITestHelper.java?rev=1182018&r1=1182017&r2=1182018&view=diff == --- hadoop/common/branches/branch-0.22/common/src/test/core/org/apache/hadoop/cli/CLITestHelper.java (original) +++ hadoop/common/branches/branch-0.22/common/src/test/core/org/apache/hadoop/cli/CLITestHelper.java Tue Oct 11 18:43:13 2011 @@ -59,7 +59,9 @@ public class CLITestHelper { public static final String TESTMODE_NOCOMPARE = "nocompare"; public static final String TEST_CACHE_DATA_DIR = System.getProperty("test.cache.data", "build/test/cache"); - + public static final String TEST_DIR_ABSOLUTE = "/tmp/testcli"; + protected static String testDirAbsolute = TEST_DIR_ABSOLUTE; + //By default, run the tests. The other mode is to run the commands and not // compare the output protected String testMode = TESTMODE_TEST; @@ -72,7 +74,7 @@ public class CLITestHelper { protected Configuration conf = null; protected String clitestDataDir = null; protected String username = null; - + /** * Read the test config file - testConfig.xml */ @@ -80,10 +82,16 @@ public class CLITestHelper { String testConfigFile = getTestFile(); if (testsFromConfigFile == null) { boolean success = false; - testConfigFile = TEST_CACHE_DATA_DIR + File.separator + testConfigFile; + String configFile = System.getProperty("test.cli.config"); + if (configFile == null) { +testConfigFile = TEST_CACHE_DATA_DIR + File.separator + testConfigFile; + } else { +testConfigFile = configFile; + } try { SAXParser p = (SAXParserFactory.newInstance()).newSAXParser(); p.parse(testConfigFile, new TestConfigFileParser()); +LOG.info("Using test config file " + testConfigFile); success = true; } catch (Exception e) { LOG.info("File: " + testConfigFile + " not found"); @@ -110,6 +118,8 @@ public class CLITestHelper { clitestDataDir = new File(TEST_CACHE_DATA_DIR). toURI().toString().replace(' ', '+'); +// Many of the tests expect a replication value of 1 in the output +conf.setInt("dfs.replication", 1); } /** @@ -128,7 +138,8 @@ public class CLITestHelper { String expCmd = cmd; expCmd = expCmd.replaceAll("CLITEST_DATA", clitestDataDir); expCmd = expCmd.replaceAll("USERNAME", username); - +expCmd = expCmd.replaceAll("TEST_DIR_ABSOLUTE", testDirAbsolute); + return expCmd; } Modified: hadoop/common/branches/branch-0.22/common/src/test/core/org/apache/hadoop/cli/util/CommandExecutor.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/common/src/test/core/org/apache/hadoop/cli/util/CommandExecutor.java?rev=1182018&r1=1182017&r2=1182018&view=diff == --- hadoop/common/branches/branch-0.22/common/src/test/core/org/apache/hadoop/cli/util/CommandExecutor.java (original) +++ hadoop/common/branches/branch-0.22/common/src/test/core/org/apache/hadoop/cli/util/CommandExecutor.java Tue Oct 11 18:43:13 2011 @@ -43,6 +43,8 @@ public abstract class CommandExecutor { args[i] = args[i].replaceAll("CLITEST_DATA", new File(CLITestHelper.TEST_CACHE_DATA_DIR). toURI().toString().replace(' ', '+')); + args[i] = args[i].replaceAll("TEST_DIR_ABSOLUTE", +CLITestHelper.TEST_DIR_ABSOLUTE); args[i] = args[i].replaceAll("USERNAME", System.getProperty("user.name")); i++;
svn commit: r1182151 - /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
Author: tucu Date: Wed Oct 12 00:27:32 2011 New Revision: 1182151 URL: http://svn.apache.org/viewvc?rev=1182151&view=rev Log: HADOOP-7642. create hadoop-dist module where TAR stitching would happen. Contributed by Thomas White. Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1182151&r1=1182150&r2=1182151&view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Wed Oct 12 00:27:32 2011 @@ -445,6 +445,9 @@ Release 0.23.0 - Unreleased HADOOP-7627. Improve MetricsAsserts to give more understandable output on failure. (todd) +HADOOP-7642. create hadoop-dist module where TAR stitching would happen. +(Thomas White via tucu) + OPTIMIZATIONS HADOOP-7333. Performance improvement in PureJavaCrc32. (Eric Caspole
svn commit: r1182151 - in /hadoop/common/trunk: hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml hadoop-dist/ hadoop-dist/pom.xml hadoop-project/pom.xml pom.xml
Author: tucu Date: Wed Oct 12 00:27:32 2011 New Revision: 1182151 URL: http://svn.apache.org/viewvc?rev=1182151&view=rev Log: HADOOP-7642. create hadoop-dist module where TAR stitching would happen. Contributed by Thomas White. Added: hadoop/common/trunk/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml hadoop/common/trunk/hadoop-dist/ hadoop/common/trunk/hadoop-dist/pom.xml Modified: hadoop/common/trunk/hadoop-project/pom.xml hadoop/common/trunk/pom.xml Added: hadoop/common/trunk/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml?rev=1182151&view=auto == --- hadoop/common/trunk/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml (added) +++ hadoop/common/trunk/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml Wed Oct 12 00:27:32 2011 @@ -0,0 +1,101 @@ +http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd";> + hadoop-mapreduce-dist + +tar.gz + + true + + + + hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/target/classes/bin + bin + +container-executor + + 0755 + + + hadoop-yarn/bin + bin + +* + + 0755 + + + bin + bin + +* + + 0755 + + + hadoop-yarn/conf + conf + +**/* + + + + sources + +**/*.jar +**/target/** + +**/bin/* +**/scripts/* + +**/dt-*/images/** + +**/file:/** +**/SecurityAuth.audit* + + +assembly/** +pom.xml +build*.xml +ivy.xml +ivy/** +INSTALL +LICENSE.txt +mr-client/** +hadoop-yarn/** +src/** + + + + sources + +**/bin/* +**/scripts/* + + 0755 + + + + + +org.apache.hadoop:hadoop-yarn-server-tests + + +modules +false +false + + + + + + false + /lib + + +org.apache.hadoop:hadoop-common +org.apache.hadoop:hadoop-hdfs + + + + Added: hadoop/common/trunk/hadoop-dist/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-dist/pom.xml?rev=1182151&view=auto == --- hadoop/common/trunk/hadoop-dist/pom.xml (added) +++ hadoop/common/trunk/hadoop-dist/pom.xml Wed Oct 12 00:27:32 2011 @@ -0,0 +1,144 @@ + + + + 4.0.0 + +org.apache.hadoop +hadoop-project +0.24.0-SNAPSHOT +../hadoop-project + + org.apache.hadoop + hadoop-dist + 0.24.0-SNAPSHOT + Apache Hadoop Distribution + Apache Hadoop Distribution + jar + + + + + org.apache.hadoop + hadoop-common + provided + + + org.apache.hadoop + hadoop-hdfs + provided + + + org.apache.hadoop + hadoop-mapreduce-client-app + provided + + + org.apache.hadoop + hadoop-yarn-api + provided + + + + + + +maven-deploy-plugin + + true + + + +org.apache.rat +apache-rat-plugin + + +pom.xml + + + + + + + + + dist + +false + + + + +org.apache.maven.plugins +maven-antrun-plugin + + +dist +package + + run + + + + + run() { +echo "\$ ${@}" +"${@}" +if [ $? != 0 ]; then + echo + echo "Failed!" + echo + exit $? +fi + } + + ROOT=`cd ${basedir}/..;pwd` + echo + echo "Current directory `pwd`" + echo + run rm -rf hadoop-${project.version} + run mkdir hadoop-${project.version} + run cd hadoop-${project.version} + run tar xzf $ROOT/hadoop-common-project/hadoop-common/target/hadoop-common-${project.version}.tar.gz --strip-compone
svn commit: r1182189 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/fs/Path.java
Author: harsh Date: Wed Oct 12 03:19:01 2011 New Revision: 1182189 URL: http://svn.apache.org/viewvc?rev=1182189&view=rev Log: HADOOP-6490. Use StringUtils over String#replace in Path#normalizePath. Contributed by Uma Maheswara Rao G. Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1182189&r1=1182188&r2=1182189&view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Wed Oct 12 03:19:01 2011 @@ -59,6 +59,9 @@ Trunk (unreleased changes) HADOOP-7721. Add log before login in KerberosAuthenticationHandler. (jitendra) +HADOOP-6490. Use StringUtils over String#replace in Path#normalizePath. +(Uma Maheswara Rao G via harsh) + Release 0.23.0 - Unreleased INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java?rev=1182189&r1=1182188&r2=1182189&view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java Wed Oct 12 03:19:01 2011 @@ -18,10 +18,12 @@ package org.apache.hadoop.fs; -import java.net.*; -import java.io.*; -import org.apache.avro.reflect.Stringable; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import org.apache.avro.reflect.Stringable; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -157,8 +159,8 @@ public class Path implements Comparable private String normalizePath(String path) { // remove double slashes & backslashes -path = path.replace("//", "/"); -path = path.replace("\\", "/"); +path = StringUtils.replace(path, "//", "/"); +path = StringUtils.replace(path, "\\", "/"); // trim trailing slash from non-root path (ignoring windows drive) int minLength = hasWindowsDrive(path, true) ? 4 : 1;
svn commit: r1182214 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/fs/Path.java
Author: harsh Date: Wed Oct 12 06:20:23 2011 New Revision: 1182214 URL: http://svn.apache.org/viewvc?rev=1182214&view=rev Log: HADOOP-7736. Remove duplicate Path#normalizePath call. Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1182214&r1=1182213&r2=1182214&view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Wed Oct 12 06:20:23 2011 @@ -30,6 +30,8 @@ Trunk (unreleased changes) HADOOP-7717. Move handling of concurrent client fail-overs to RetryInvocationHandler (atm) +HADOOP-7736. Remove duplicate Path#normalizePath call. (harsh) + BUGS HADOOP-7606. Upgrade Jackson to version 1.7.1 to match the version required Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java?rev=1182214&r1=1182213&r2=1182214&view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java Wed Oct 12 06:20:23 2011 @@ -77,7 +77,7 @@ public class Path implements Comparable } URI resolved = parentUri.resolve(child.uri); initialize(resolved.getScheme(), resolved.getAuthority(), - normalizePath(resolved.getPath()), resolved.getFragment()); + resolved.getPath(), resolved.getFragment()); } private void checkPathArg( String path ) {
svn commit: r1182217 - /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
Author: harsh Date: Wed Oct 12 06:24:04 2011 New Revision: 1182217 URL: http://svn.apache.org/viewvc?rev=1182217&view=rev Log: Move the HADOOP-6490 Changes note to the right section. Accidentally placed it under Bugs in r1182189. Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1182217&r1=1182216&r2=1182217&view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Wed Oct 12 06:24:04 2011 @@ -30,6 +30,9 @@ Trunk (unreleased changes) HADOOP-7717. Move handling of concurrent client fail-overs to RetryInvocationHandler (atm) +HADOOP-6490. Use StringUtils over String#replace in Path#normalizePath. + (Uma Maheswara Rao G via harsh) + HADOOP-7736. Remove duplicate Path#normalizePath call. (harsh) BUGS @@ -61,9 +64,6 @@ Trunk (unreleased changes) HADOOP-7721. Add log before login in KerberosAuthenticationHandler. (jitendra) -HADOOP-6490. Use StringUtils over String#replace in Path#normalizePath. -(Uma Maheswara Rao G via harsh) - Release 0.23.0 - Unreleased INCOMPATIBLE CHANGES