Author: rvs
Date: Wed Dec 14 00:27:53 2011
New Revision: 1214001
URL: http://svn.apache.org/viewvc?rev=1214001&view=rev
Log:
Updating PIG's patch
Modified:
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/pig/patch
Modified:
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/pig/patch
URL:
http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/pig/patch?rev=1214001&r1=1214000&r2=1214001&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/pig/patch
(original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/pig/patch
Wed Dec 14 00:27:53 2011
@@ -1,98 +1,185 @@
-Index: shims/test/hadoop23/org/apache/pig/test/MiniCluster.java
-===================================================================
---- shims/test/hadoop23/org/apache/pig/test/MiniCluster.java (revision
1208121)
-+++ shims/test/hadoop23/org/apache/pig/test/MiniCluster.java (working copy)
-@@ -92,14 +92,16 @@
- e.printStackTrace();
- }
- */
-- m_dfs.getFileSystem().copyFromLocalFile(new
Path("file:///hadoop-mapreduce-client-app-0.23.0-SNAPSHOT.jar"), new
Path("/hadoop-mapreduce-client-app-0.23.0-SNAPSHOT.jar"));
-- m_dfs.getFileSystem().copyFromLocalFile(new
Path("file:///hadoop-mapreduce-client-jobclient-0.23.0-SNAPSHOT.jar"), new
Path("/hadoop-mapreduce-client-jobclient-0.23.0-SNAPSHOT.jar"));
-- m_dfs.getFileSystem().copyFromLocalFile(new
Path("file:///pig.jar"), new Path("/pig.jar"));
-- m_dfs.getFileSystem().copyFromLocalFile(new
Path("file:///pig-test.jar"), new Path("/pig-test.jar"));
-+ File jarBaseDir = new File("./build/ivy/lib/Pig");
-+
-+ m_dfs.getFileSystem().copyFromLocalFile(new Path(new
File(jarBaseDir, "hadoop-mapreduce-client-app-0.23.0-SNAPSHOT.jar").toURI()),
new Path("/hadoop-mapreduce-client-app-0.23.0-SNAPSHOT.jar"));
-+ m_dfs.getFileSystem().copyFromLocalFile(new Path(new
File(jarBaseDir,
"hadoop-mapreduce-client-jobclient-0.23.0-SNAPSHOT.jar").toURI()), new
Path("/hadoop-mapreduce-client-jobclient-0.23.0-SNAPSHOT.jar"));
-+ m_dfs.getFileSystem().copyFromLocalFile(new Path(new
File("pig-withouthadoop.jar").toURI()), new Path("/pig-withouthadoop.jar"));
-+ //m_dfs.getFileSystem().copyFromLocalFile(new Path(new
File(jarBaseDir, "pig-test.jar").toURI()), new Path("/pig-test.jar"));
+diff --git build.xml build.xml
+index 1edc448..e8af7ce 100644
+--- build.xml
++++ build.xml
+@@ -156,7 +156,7 @@
+ input="${hadoop-core.version}"
+ regexp="\d+\.(\d+)\.\d+"
+ select="\1" /-->
+- <property name="hadoopversion" value="20" />
++ <property name="hadoopversion" value="23" />
- DistributedCache.addFileToClassPath(new
Path("/hadoop-mapreduce-client-app-0.23.0-SNAPSHOT.jar"), m_conf);
-- DistributedCache.addFileToClassPath(new Path("/pig.jar"), m_conf);
-- DistributedCache.addFileToClassPath(new Path("/pig-test.jar"),
m_conf);
-+ DistributedCache.addFileToClassPath(new
Path("/pig-withouthadoop.jar"), m_conf);
-+ //DistributedCache.addFileToClassPath(new Path("/pig-test.jar"),
m_conf);
- DistributedCache.addFileToClassPath(new
Path("/hadoop-mapreduce-client-jobclient-0.23.0-SNAPSHOT.jar"), m_conf);
- String cachefile = m_conf.get("mapreduce.job.cache.files");
- m_conf.set("alternative.mapreduce.job.cache.files", cachefile);
-Index: ivy.xml
-===================================================================
---- ivy.xml 2011-11-21 17:26:11.000000000 -0800
-+++ ivy.xml 2011-12-02 20:49:31.149818867 -0800
-@@ -67,41 +67,50 @@
- <dependency org="org.apache.hadoop" name="hadoop-test"
rev="${hadoop-test.version}"
- conf="hadoop20->default"/>
- <dependency org="org.apache.hadoop" name="hadoop-annotations"
-- rev="${hadoop-common.version}" conf="hadoop23->master"/>
-+ rev="${hadoop-common.version}" conf="hadoop23->default"/>
- <dependency org="org.apache.hadoop" name="hadoop-auth"
-- rev="${hadoop-common.version}" conf="hadoop23->master"/>
-+ rev="${hadoop-common.version}" conf="hadoop23->default"/>
- <dependency org="org.apache.hadoop" name="hadoop-common"
-- rev="${hadoop-common.version}" conf="hadoop23->master">
-+ rev="${hadoop-common.version}" conf="hadoop23->default">
- <artifact name="hadoop-common" ext="jar" />
- <artifact name="hadoop-common" type="tests" ext="jar"
m:classifier="tests" />
- </dependency>
- <dependency org="org.apache.hadoop" name="hadoop-hdfs"
-- rev="${hadoop-hdfs.version}" conf="hadoop23->master">
-+ rev="${hadoop-hdfs.version}" conf="hadoop23->default">
- <artifact name="hadoop-hdfs" ext="jar" />
- <artifact name="hadoop-hdfs" type="tests" ext="jar"
m:classifier="tests" />
-+ <exclude org="commons-daemon" module="commons-daemon"/><!--bad POM-->
-+ <exclude org="org.apache.commons" module="commons-daemon"/><!--bad
POM-->
- </dependency>
-+ <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-app"
rev="${hadoop-mapreduce.version}"
-+ conf="hadoop23->default"/>
-+ <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-common"
-+ rev="${hadoop-mapreduce.version}" conf="hadoop23->default"/>
- <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-core"
rev="${hadoop-mapreduce.version}"
-- conf="hadoop23->master"/>
-+ conf="hadoop23->default"/>
-+ <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-hs"
rev="${hadoop-mapreduce.version}"
-+ conf="hadoop23->default"/>
+ <condition property="isHadoop23">
+ <equals arg1="hadoopversion" arg2="23"/>
+@@ -415,6 +415,10 @@
+ </antcall>
+
+ <copy file="${basedir}/test/hbase-site.xml"
tofile="${test.build.classes}/hbase-site.xml"/>
++
++ <ivy:cachepath pathid="mr-apps-test.classpath" />
++ <property name="mr-apps-classpath" refid="mr-apps-test.classpath" />
++ <echo file="${test.build.classes}/mrapp-generated-classpath"
message="${mr-apps-classpath}" />
+ </target>
+
+ <!-- This target is for default compilation -->
+diff --git ivy.xml ivy.xml
+index 9fc2f94..85803c5 100644
+--- ivy.xml
++++ ivy.xml
+@@ -60,6 +60,24 @@
+ conf="hadoop23->master"/>
+ <dependency org="commons-configuration" name="commons-configuration"
rev="${commons-configuration.version}"
+ conf="hadoop23->master"/>
++ <dependency org="commons-collections" name="commons-collections"
rev="${commons-collections.version}"
++ conf="hadoop23->master"/>
++ <dependency org="javax.servlet" name="servlet-api"
rev="${servlet-api.version}"
++ conf="hadoop23->master"/>
++ <dependency org="javax.ws.rs" name="jsr311-api"
rev="${jsr311-api.version}"
++ conf="hadoop23->master"/>
++ <dependency org="org.mortbay.jetty" name="jetty" rev="${jetty.version}"
++ conf="hadoop23->master"/>
++ <dependency org="com.google.protobuf" name="protobuf-java"
rev="${protobuf-java.version}"
++ conf="hadoop23->master"/>
++ <dependency org="org.mortbay.jetty" name="jetty-util"
rev="${jetty-util.version}"
++ conf="hadoop23->master"/>
++ <dependency org="com.google.inject" name="guice" rev="${guice.version}"
++ conf="hadoop23->master"/>
++ <dependency org="com.google.inject.extensions" name="guice-servlet"
rev="${guice-servlet.version}"
++ conf="hadoop23->master"/>
++ <dependency org="aopalliance" name="aopalliance"
rev="${aopalliance.version}"
++ conf="hadoop23->master"/>
+ <dependency org="log4j" name="log4j" rev="${log4j.version}"
+ conf="compile->master"/>
+ <dependency org="org.apache.hadoop" name="hadoop-core"
rev="${hadoop-core.version}"
+@@ -84,7 +102,8 @@
+ conf="hadoop23->master"/>
<dependency org="org.apache.hadoop"
name="hadoop-mapreduce-client-jobclient" rev="${hadoop-mapreduce.version}"
-- conf="hadoop23->master">
-+ conf="hadoop23->default">
-+ <artifact name="hadoop-mapreduce-client-jobclient" type="jar"/>
- <artifact name="hadoop-mapreduce-client-jobclient" type="jar"
m:classifier="tests"/>
+ conf="hadoop23->master">
+- <artifact name="hadoop-mapreduce-client-jobclient" type="jar"
m:classifier="tests"/>
++ <artifact name="hadoop-mapreduce-client-jobclient" ext="jar" />
++ <artifact name="hadoop-mapreduce-client-jobclient" type="tests"
ext="jar" m:classifier="tests"/>
<exclude org="commons-daemon" module="commons-daemon"/><!--bad POM-->
<exclude org="org.apache.commons" module="commons-daemon"/><!--bad
POM-->
</dependency>
-- <dependency org="org.apache.hadoop" name="hadoop-yarn-server-tests"
rev="${hadoop-mapreduce.version}"
-- conf="hadoop23->master">
-- <artifact name="hadoop-yarn-server-tests" type="jar"
m:classifier="tests"/>
-- </dependency>
-- <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-common"
-- rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
-+ <dependency org="org.apache.hadoop"
name="hadoop-mapreduce-client-shuffle"
-+ rev="${hadoop-mapreduce.version}" conf="hadoop23->default"/>
+@@ -92,16 +111,28 @@
+ conf="hadoop23->master">
+ <artifact name="hadoop-yarn-server-tests" type="jar"
m:classifier="tests"/>
+ </dependency>
++ <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-app"
rev="${hadoop-mapreduce.version}"
++ conf="hadoop23->master" />
++ <dependency org="org.apache.hadoop"
name="hadoop-mapreduce-client-shuffle" rev="${hadoop-mapreduce.version}"
++ conf="hadoop23->master" />
+ <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-common"
+ rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
<dependency org="org.apache.hadoop" name="hadoop-yarn-api"
- rev="${hadoop-common.version}" conf="hadoop23->master"/>
-+ rev="${hadoop-common.version}" conf="hadoop23->default"/>
++ rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
<dependency org="org.apache.hadoop" name="hadoop-yarn-common"
- rev="${hadoop-common.version}" conf="hadoop23->master"/>
-+ rev="${hadoop-common.version}" conf="hadoop23->default"/>
++ rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
++ <dependency org="org.apache.hadoop" name="hadoop-yarn-server"
++ rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
++ <dependency org="org.apache.hadoop" name="hadoop-yarn-server-web-proxy"
++ rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
++ <dependency org="org.apache.hadoop" name="hadoop-yarn-server-common"
++ rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
<dependency org="org.apache.hadoop" name="hadoop-yarn-server-nodemanager"
- rev="${hadoop-common.version}" conf="hadoop23->master"/>
-+ rev="${hadoop-common.version}" conf="hadoop23->default"/>
++ rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
<dependency org="org.apache.hadoop"
name="hadoop-yarn-server-resourcemanager"
- rev="${hadoop-common.version}" conf="hadoop23->master"/>
-+ rev="${hadoop-common.version}" conf="hadoop23->default"/>
-+ <dependency org="org.apache.hadoop" name="hadoop-yarn-server-tests"
rev="${hadoop-mapreduce.version}"
-+ conf="hadoop23->master">
-+ <artifact name="hadoop-yarn-server-tests" type="jar"
m:classifier="tests"/>
-+ </dependency>
++ rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
++ <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-hs"
++ rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
<dependency org="org.mortbay.jetty" name="jetty" rev="${jetty.version}"
conf="compile->master"/>
<dependency org="org.mortbay.jetty" name="jetty-util"
rev="${jetty-util.version}"
+diff --git ivy/libraries.properties ivy/libraries.properties
+index 5af5c3a..f0deea9 100644
+--- ivy/libraries.properties
++++ ivy/libraries.properties
+@@ -24,14 +24,15 @@ commons-logging.version=1.1.1
+ commons-lang.version=2.4
+ commons-configuration.version=1.6
+ commons-httpclient.version=3.1
++commons-collections.version=3.2.1
+ checkstyle.version=4.2
+ ivy.version=2.2.0
+ guava.version=r06
+ hadoop-core.version=0.20.2
+ hadoop-test.version=0.20.2
+-hadoop-common.version=0.23.0-SNAPSHOT
+-hadoop-hdfs.version=0.23.0-SNAPSHOT
+-hadoop-mapreduce.version=0.23.0-SNAPSHOT
++hadoop-common.version=0.23.1-SNAPSHOT
++hadoop-hdfs.version=0.23.1-SNAPSHOT
++hadoop-mapreduce.version=0.23.1-SNAPSHOT
+ hbase.version=0.90.0
+ hsqldb.version=1.8.0.10
+ jackson.version=1.7.3
+@@ -57,3 +58,10 @@ slf4j-log4j12.version=1.6.1
+ xerces.version=1.4.4
+ wagon-http.version=1.0-beta-2
+ zookeeper.version=3.3.3
++servlet.version=4.0.6
++servlet-api.version=2.5
++protobuf-java.version=2.4.0a
++guice.version=2.0
++guice-servlet.version=2.0
++aopalliance.version=1.0
++jsr311-api.version=1.1.1
+diff --git shims/test/hadoop23/org/apache/pig/test/MiniCluster.java
shims/test/hadoop23/org/apache/pig/test/MiniCluster.java
+index 61f6372..8c3f468 100644
+--- shims/test/hadoop23/org/apache/pig/test/MiniCluster.java
++++ shims/test/hadoop23/org/apache/pig/test/MiniCluster.java
+@@ -72,7 +72,7 @@ public class MiniCluster extends MiniGenericCluster {
+ m_dfs_conf = m_dfs.getConfiguration(0);
+
+ m_mr = new MiniMRYarnCluster("PigMiniCluster");
+- m_mr.init(new Configuration());
++ m_mr.init(config);
+ //m_mr.init(m_dfs_conf);
+ m_mr.start();
+
+@@ -82,27 +82,6 @@ public class MiniCluster extends MiniGenericCluster {
+
+ m_conf = m_mr_conf;
+ m_conf.set("fs.default.name",
m_dfs_conf.get("fs.default.name"));
+-
+- /*
+- try {
+- DistributedCache.addCacheFile(new
URI("file:///hadoop-mapreduce-client-app-1.0-SNAPSHOT.jar"), m_conf);
+- DistributedCache.addCacheFile(new
URI("file:///hadoop-mapreduce-client-jobclient-1.0-SNAPSHOT.jar"), m_conf);
+- DistributedCache.addCacheFile(new
URI("file:///pig.jar"), m_conf);
+- } catch (Exception e) {
+- e.printStackTrace();
+- }
+- */
+- m_dfs.getFileSystem().copyFromLocalFile(new
Path("file:///hadoop-mapreduce-client-app-0.23.0-SNAPSHOT.jar"), new
Path("/hadoop-mapreduce-client-app-0.23.0-SNAPSHOT.jar"));
+- m_dfs.getFileSystem().copyFromLocalFile(new
Path("file:///hadoop-mapreduce-client-jobclient-0.23.0-SNAPSHOT.jar"), new
Path("/hadoop-mapreduce-client-jobclient-0.23.0-SNAPSHOT.jar"));
+- m_dfs.getFileSystem().copyFromLocalFile(new
Path("file:///pig.jar"), new Path("/pig.jar"));
+- m_dfs.getFileSystem().copyFromLocalFile(new
Path("file:///pig-test.jar"), new Path("/pig-test.jar"));
+-
+- DistributedCache.addFileToClassPath(new
Path("/hadoop-mapreduce-client-app-0.23.0-SNAPSHOT.jar"), m_conf);
+- DistributedCache.addFileToClassPath(new Path("/pig.jar"), m_conf);
+- DistributedCache.addFileToClassPath(new Path("/pig-test.jar"),
m_conf);
+- DistributedCache.addFileToClassPath(new
Path("/hadoop-mapreduce-client-jobclient-0.23.0-SNAPSHOT.jar"), m_conf);
+- String cachefile = m_conf.get("mapreduce.job.cache.files");
+- m_conf.set("alternative.mapreduce.job.cache.files", cachefile);
+ m_conf.unset("mapreduce.job.cache.files");
+
+ //ConfigurationUtil.mergeConf(m_conf, m_dfs_conf);
+diff --git src/org/apache/pig/tools/pigstats/PigStatsUtil.java
src/org/apache/pig/tools/pigstats/PigStatsUtil.java
+index 289159c..13eafa5 100644
+--- src/org/apache/pig/tools/pigstats/PigStatsUtil.java
++++ src/org/apache/pig/tools/pigstats/PigStatsUtil.java
+@@ -51,7 +51,8 @@ public abstract class PigStatsUtil {
+ public static final String TASK_COUNTER_GROUP
+ = "org.apache.hadoop.mapred.Task$Counter";
+ public static final String FS_COUNTER_GROUP
+- = "FileSystemCounters";
++ = "org.apache.hadoop.mapreduce.FileSystemCounter";
++ //= "FileSystemCounters";
+ public static final String MAP_INPUT_RECORDS
+ = "MAP_INPUT_RECORDS";
+ public static final String MAP_OUTPUT_RECORDS
