[Hadoop Wiki] Update of "HadoopMeetupAtApacheCon" by KonstantinShvachko
Dear Wiki user, You have subscribed to a wiki page or wiki category on "Hadoop Wiki" for change notification. The "HadoopMeetupAtApacheCon" page has been changed by KonstantinShvachko: http://wiki.apache.org/hadoop/HadoopMeetupAtApacheCon?action=diff&rev1=5&rev2=6 (Please add your name here if you plan to attend.) 1. Owen O'Malley - 2. Milind Bhandarkar + 1. Milind Bhandarkar - 3. Allen Wittenauer + 1. Allen Wittenauer + 1. Konstantin Shvachko
svn commit: r1182018 - in /hadoop/common/branches/branch-0.22/common: CHANGES.txt src/test/core/org/apache/hadoop/cli/CLITestHelper.java src/test/core/org/apache/hadoop/cli/util/CommandExecutor.java
Author: cos
Date: Tue Oct 11 18:43:13 2011
New Revision: 1182018
URL: http://svn.apache.org/viewvc?rev=1182018&view=rev
Log:
HADOOP-7730. Allow TestCLI to be run against a cluster. Contributed by Tom
White, Konstantin Boudnik.
Modified:
hadoop/common/branches/branch-0.22/common/CHANGES.txt
hadoop/common/branches/branch-0.22/common/src/test/core/org/apache/hadoop/cli/CLITestHelper.java
hadoop/common/branches/branch-0.22/common/src/test/core/org/apache/hadoop/cli/util/CommandExecutor.java
Modified: hadoop/common/branches/branch-0.22/common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/common/CHANGES.txt?rev=1182018&r1=1182017&r2=1182018&view=diff
==
--- hadoop/common/branches/branch-0.22/common/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.22/common/CHANGES.txt Tue Oct 11 18:43:13
2011
@@ -549,6 +549,8 @@ Release 0.22.0 - Unreleased
HADOOP-7697. Remove ivy dependency on different version of slf4j in avro.
(shv)
+HADOOP-7730. Allow TestCLI to be run against a cluster (cos)
+
Release 0.21.1 - Unreleased
IMPROVEMENTS
Modified:
hadoop/common/branches/branch-0.22/common/src/test/core/org/apache/hadoop/cli/CLITestHelper.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/common/src/test/core/org/apache/hadoop/cli/CLITestHelper.java?rev=1182018&r1=1182017&r2=1182018&view=diff
==
---
hadoop/common/branches/branch-0.22/common/src/test/core/org/apache/hadoop/cli/CLITestHelper.java
(original)
+++
hadoop/common/branches/branch-0.22/common/src/test/core/org/apache/hadoop/cli/CLITestHelper.java
Tue Oct 11 18:43:13 2011
@@ -59,7 +59,9 @@ public class CLITestHelper {
public static final String TESTMODE_NOCOMPARE = "nocompare";
public static final String TEST_CACHE_DATA_DIR =
System.getProperty("test.cache.data", "build/test/cache");
-
+ public static final String TEST_DIR_ABSOLUTE = "/tmp/testcli";
+ protected static String testDirAbsolute = TEST_DIR_ABSOLUTE;
+
//By default, run the tests. The other mode is to run the commands and not
// compare the output
protected String testMode = TESTMODE_TEST;
@@ -72,7 +74,7 @@ public class CLITestHelper {
protected Configuration conf = null;
protected String clitestDataDir = null;
protected String username = null;
-
+
/**
* Read the test config file - testConfig.xml
*/
@@ -80,10 +82,16 @@ public class CLITestHelper {
String testConfigFile = getTestFile();
if (testsFromConfigFile == null) {
boolean success = false;
- testConfigFile = TEST_CACHE_DATA_DIR + File.separator + testConfigFile;
+ String configFile = System.getProperty("test.cli.config");
+ if (configFile == null) {
+testConfigFile = TEST_CACHE_DATA_DIR + File.separator + testConfigFile;
+ } else {
+testConfigFile = configFile;
+ }
try {
SAXParser p = (SAXParserFactory.newInstance()).newSAXParser();
p.parse(testConfigFile, new TestConfigFileParser());
+LOG.info("Using test config file " + testConfigFile);
success = true;
} catch (Exception e) {
LOG.info("File: " + testConfigFile + " not found");
@@ -110,6 +118,8 @@ public class CLITestHelper {
clitestDataDir = new File(TEST_CACHE_DATA_DIR).
toURI().toString().replace(' ', '+');
+// Many of the tests expect a replication value of 1 in the output
+conf.setInt("dfs.replication", 1);
}
/**
@@ -128,7 +138,8 @@ public class CLITestHelper {
String expCmd = cmd;
expCmd = expCmd.replaceAll("CLITEST_DATA", clitestDataDir);
expCmd = expCmd.replaceAll("USERNAME", username);
-
+expCmd = expCmd.replaceAll("TEST_DIR_ABSOLUTE", testDirAbsolute);
+
return expCmd;
}
Modified:
hadoop/common/branches/branch-0.22/common/src/test/core/org/apache/hadoop/cli/util/CommandExecutor.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/common/src/test/core/org/apache/hadoop/cli/util/CommandExecutor.java?rev=1182018&r1=1182017&r2=1182018&view=diff
==
---
hadoop/common/branches/branch-0.22/common/src/test/core/org/apache/hadoop/cli/util/CommandExecutor.java
(original)
+++
hadoop/common/branches/branch-0.22/common/src/test/core/org/apache/hadoop/cli/util/CommandExecutor.java
Tue Oct 11 18:43:13 2011
@@ -43,6 +43,8 @@ public abstract class CommandExecutor {
args[i] = args[i].replaceAll("CLITEST_DATA",
new File(CLITestHelper.TEST_CACHE_DATA_DIR).
toURI().toString().replace(' ', '+'));
+ args[i] = args[i].replaceAll("TEST_DIR_ABSOLUTE",
+CLITestHelper.TEST_DIR_ABSOLUTE);
args[i] = args[i].replaceAll("USERNAME",
System.getProperty("user.name"));
i++;
svn commit: r1182151 - /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
Author: tucu Date: Wed Oct 12 00:27:32 2011 New Revision: 1182151 URL: http://svn.apache.org/viewvc?rev=1182151&view=rev Log: HADOOP-7642. create hadoop-dist module where TAR stitching would happen. Contributed by Thomas White. Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1182151&r1=1182150&r2=1182151&view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Wed Oct 12 00:27:32 2011 @@ -445,6 +445,9 @@ Release 0.23.0 - Unreleased HADOOP-7627. Improve MetricsAsserts to give more understandable output on failure. (todd) +HADOOP-7642. create hadoop-dist module where TAR stitching would happen. +(Thomas White via tucu) + OPTIMIZATIONS HADOOP-7333. Performance improvement in PureJavaCrc32. (Eric Caspole
svn commit: r1182151 - in /hadoop/common/trunk: hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml hadoop-dist/ hadoop-dist/pom.xml hadoop-project/pom.xml pom.xml
Author: tucu
Date: Wed Oct 12 00:27:32 2011
New Revision: 1182151
URL: http://svn.apache.org/viewvc?rev=1182151&view=rev
Log:
HADOOP-7642. create hadoop-dist module where TAR stitching would happen.
Contributed by Thomas White.
Added:
hadoop/common/trunk/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml
hadoop/common/trunk/hadoop-dist/
hadoop/common/trunk/hadoop-dist/pom.xml
Modified:
hadoop/common/trunk/hadoop-project/pom.xml
hadoop/common/trunk/pom.xml
Added:
hadoop/common/trunk/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml?rev=1182151&view=auto
==
---
hadoop/common/trunk/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml
(added)
+++
hadoop/common/trunk/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml
Wed Oct 12 00:27:32 2011
@@ -0,0 +1,101 @@
+http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0
http://maven.apache.org/xsd/assembly-1.1.0.xsd";>
+ hadoop-mapreduce-dist
+
+tar.gz
+
+ true
+
+
+
+
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/target/classes/bin
+ bin
+
+container-executor
+
+ 0755
+
+
+ hadoop-yarn/bin
+ bin
+
+*
+
+ 0755
+
+
+ bin
+ bin
+
+*
+
+ 0755
+
+
+ hadoop-yarn/conf
+ conf
+
+**/*
+
+
+
+ sources
+
+**/*.jar
+**/target/**
+
+**/bin/*
+**/scripts/*
+
+**/dt-*/images/**
+
+**/file:/**
+**/SecurityAuth.audit*
+
+
+assembly/**
+pom.xml
+build*.xml
+ivy.xml
+ivy/**
+INSTALL
+LICENSE.txt
+mr-client/**
+hadoop-yarn/**
+src/**
+
+
+
+ sources
+
+**/bin/*
+**/scripts/*
+
+ 0755
+
+
+
+
+
+org.apache.hadoop:hadoop-yarn-server-tests
+
+
+modules
+false
+false
+
+
+
+
+
+ false
+ /lib
+
+
+org.apache.hadoop:hadoop-common
+org.apache.hadoop:hadoop-hdfs
+
+
+
+
Added: hadoop/common/trunk/hadoop-dist/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-dist/pom.xml?rev=1182151&view=auto
==
--- hadoop/common/trunk/hadoop-dist/pom.xml (added)
+++ hadoop/common/trunk/hadoop-dist/pom.xml Wed Oct 12 00:27:32 2011
@@ -0,0 +1,144 @@
+
+
+
+ 4.0.0
+
+org.apache.hadoop
+hadoop-project
+0.24.0-SNAPSHOT
+../hadoop-project
+
+ org.apache.hadoop
+ hadoop-dist
+ 0.24.0-SNAPSHOT
+ Apache Hadoop Distribution
+ Apache Hadoop Distribution
+ jar
+
+
+
+
+ org.apache.hadoop
+ hadoop-common
+ provided
+
+
+ org.apache.hadoop
+ hadoop-hdfs
+ provided
+
+
+ org.apache.hadoop
+ hadoop-mapreduce-client-app
+ provided
+
+
+ org.apache.hadoop
+ hadoop-yarn-api
+ provided
+
+
+
+
+
+
+maven-deploy-plugin
+
+ true
+
+
+
+org.apache.rat
+apache-rat-plugin
+
+
+pom.xml
+
+
+
+
+
+
+
+
+ dist
+
+false
+
+
+
+
+org.apache.maven.plugins
+maven-antrun-plugin
+
+
+dist
+package
+
+ run
+
+
+
+
+ run() {
+echo "\$ ${@}"
+"${@}"
+if [ $? != 0 ]; then
+ echo
+ echo "Failed!"
+ echo
+ exit $?
+fi
+ }
+
+ ROOT=`cd ${basedir}/..;pwd`
+ echo
+ echo "Current directory `pwd`"
+ echo
+ run rm -rf hadoop-${project.version}
+ run mkdir hadoop-${project.version}
+ run cd hadoop-${project.version}
+ run tar xzf
$ROOT/hadoop-common-project/hadoop-common/target/hadoop-common-${project.version}.tar.gz
--strip-compone
svn commit: r1182189 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/fs/Path.java
Author: harsh
Date: Wed Oct 12 03:19:01 2011
New Revision: 1182189
URL: http://svn.apache.org/viewvc?rev=1182189&view=rev
Log:
HADOOP-6490. Use StringUtils over String#replace in Path#normalizePath.
Contributed by Uma Maheswara Rao G.
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1182189&r1=1182188&r2=1182189&view=diff
==
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Wed Oct
12 03:19:01 2011
@@ -59,6 +59,9 @@ Trunk (unreleased changes)
HADOOP-7721. Add log before login in KerberosAuthenticationHandler.
(jitendra)
+HADOOP-6490. Use StringUtils over String#replace in Path#normalizePath.
+(Uma Maheswara Rao G via harsh)
+
Release 0.23.0 - Unreleased
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java?rev=1182189&r1=1182188&r2=1182189&view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
Wed Oct 12 03:19:01 2011
@@ -18,10 +18,12 @@
package org.apache.hadoop.fs;
-import java.net.*;
-import java.io.*;
-import org.apache.avro.reflect.Stringable;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import org.apache.avro.reflect.Stringable;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@@ -157,8 +159,8 @@ public class Path implements Comparable
private String normalizePath(String path) {
// remove double slashes & backslashes
-path = path.replace("//", "/");
-path = path.replace("\\", "/");
+path = StringUtils.replace(path, "//", "/");
+path = StringUtils.replace(path, "\\", "/");
// trim trailing slash from non-root path (ignoring windows drive)
int minLength = hasWindowsDrive(path, true) ? 4 : 1;
svn commit: r1182214 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/fs/Path.java
Author: harsh
Date: Wed Oct 12 06:20:23 2011
New Revision: 1182214
URL: http://svn.apache.org/viewvc?rev=1182214&view=rev
Log:
HADOOP-7736. Remove duplicate Path#normalizePath call.
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1182214&r1=1182213&r2=1182214&view=diff
==
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Wed Oct
12 06:20:23 2011
@@ -30,6 +30,8 @@ Trunk (unreleased changes)
HADOOP-7717. Move handling of concurrent client fail-overs to
RetryInvocationHandler (atm)
+HADOOP-7736. Remove duplicate Path#normalizePath call. (harsh)
+
BUGS
HADOOP-7606. Upgrade Jackson to version 1.7.1 to match the version required
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java?rev=1182214&r1=1182213&r2=1182214&view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
Wed Oct 12 06:20:23 2011
@@ -77,7 +77,7 @@ public class Path implements Comparable
}
URI resolved = parentUri.resolve(child.uri);
initialize(resolved.getScheme(), resolved.getAuthority(),
- normalizePath(resolved.getPath()), resolved.getFragment());
+ resolved.getPath(), resolved.getFragment());
}
private void checkPathArg( String path ) {
svn commit: r1182217 - /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
Author: harsh Date: Wed Oct 12 06:24:04 2011 New Revision: 1182217 URL: http://svn.apache.org/viewvc?rev=1182217&view=rev Log: Move the HADOOP-6490 Changes note to the right section. Accidentally placed it under Bugs in r1182189. Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1182217&r1=1182216&r2=1182217&view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Wed Oct 12 06:24:04 2011 @@ -30,6 +30,9 @@ Trunk (unreleased changes) HADOOP-7717. Move handling of concurrent client fail-overs to RetryInvocationHandler (atm) +HADOOP-6490. Use StringUtils over String#replace in Path#normalizePath. + (Uma Maheswara Rao G via harsh) + HADOOP-7736. Remove duplicate Path#normalizePath call. (harsh) BUGS @@ -61,9 +64,6 @@ Trunk (unreleased changes) HADOOP-7721. Add log before login in KerberosAuthenticationHandler. (jitendra) -HADOOP-6490. Use StringUtils over String#replace in Path#normalizePath. -(Uma Maheswara Rao G via harsh) - Release 0.23.0 - Unreleased INCOMPATIBLE CHANGES
