CRUNCH-16: Fix dependency versions. Move all dependency versions to properties for easier management. Align Crunch's dependencies with those on Hadoop's classpath. Remove Scrunch's last non-test dependency on commons-io. Add HBase warning note to README.
Signed-off-by: Josh Wills <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-crunch/commit/9d8b679e Tree: http://git-wip-us.apache.org/repos/asf/incubator-crunch/tree/9d8b679e Diff: http://git-wip-us.apache.org/repos/asf/incubator-crunch/diff/9d8b679e Branch: refs/heads/master Commit: 9d8b679ec0bbd007e65c33f2018b2c9630cfc700 Parents: 5d9b698 Author: Matthias Friedrich <[email protected]> Authored: Thu Aug 16 20:47:19 2012 +0200 Committer: Josh Wills <[email protected]> Committed: Tue Aug 21 19:46:28 2012 -0700 ---------------------------------------------------------------------- README | 6 ++ crunch-hbase/pom.xml | 7 ++ .../scrunch/interpreter/InterpreterRunner.scala | 4 +- crunch-test/pom.xml | 6 ++ crunch/pom.xml | 11 ++- pom.xml | 59 ++++++++++----- 6 files changed, 68 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/9d8b679e/README ---------------------------------------------------------------------- diff --git a/README b/README index d061cab..b80235d 100644 --- a/README +++ b/README @@ -24,3 +24,9 @@ local Maven cache: mvn install +Crunch has experimental support for Hadoop 2 through the "hadoop-2" build +profile (add -Phadoop-2 to enable it). If you want to use HBase support on +Hadoop 2, please note that you have to build HBase 0.94.1 from source using +the following command: + + mvn clean install -Dhadoop.profile=2.0 http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/9d8b679e/crunch-hbase/pom.xml ---------------------------------------------------------------------- diff --git a/crunch-hbase/pom.xml b/crunch-hbase/pom.xml index 3bd6e54..d97b457 100644 --- a/crunch-hbase/pom.xml +++ b/crunch-hbase/pom.xml @@ -68,6 +68,13 @@ under the License. <type>test-jar</type> <scope>test</scope> </dependency> + + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + <scope>test</scope> + </dependency> + </dependencies> <build> http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/9d8b679e/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/interpreter/InterpreterRunner.scala ---------------------------------------------------------------------- diff --git a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/interpreter/InterpreterRunner.scala b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/interpreter/InterpreterRunner.scala index 18cb201..196e222 100644 --- a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/interpreter/InterpreterRunner.scala +++ b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/interpreter/InterpreterRunner.scala @@ -33,7 +33,7 @@ import scala.tools.nsc.io.Jar import scala.tools.nsc.io.VirtualDirectory import com.google.common.io.Files -import org.apache.commons.io.IOUtils +import com.google.common.io.Closeables import org.apache.hadoop.conf.Configuration import org.apache.crunch.util.DistCache @@ -153,7 +153,7 @@ object InterpreterRunner extends MainGenericRunner { addVirtualDirectoryToJar(virtualDirectory, "", jarStream) return tempJar } finally { - IOUtils.closeQuietly(jarStream) + Closeables.closeQuietly(jarStream) } } http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/9d8b679e/crunch-test/pom.xml ---------------------------------------------------------------------- diff --git a/crunch-test/pom.xml b/crunch-test/pom.xml index 8a2f019..a1e3899 100644 --- a/crunch-test/pom.xml +++ b/crunch-test/pom.xml @@ -48,6 +48,12 @@ under the License. <artifactId>hadoop-minicluster</artifactId> </dependency> + <!-- Required by LocalJobRunner --> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + </dependency> + <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/9d8b679e/crunch/pom.xml ---------------------------------------------------------------------- diff --git a/crunch/pom.xml b/crunch/pom.xml index 20c75b7..07951ec 100644 --- a/crunch/pom.xml +++ b/crunch/pom.xml @@ -52,23 +52,21 @@ under the License. </dependency> <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> - </dependency> - - <dependency> <groupId>commons-codec</groupId> <artifactId>commons-codec</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>org.codehaus.jackson</groupId> <artifactId>jackson-core-asl</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>org.codehaus.jackson</groupId> <artifactId>jackson-mapper-asl</artifactId> + <scope>provided</scope> </dependency> <!-- Both Protobufs and Thrift are supported as @@ -109,16 +107,19 @@ under the License. <dependency> <groupId>commons-logging</groupId> <artifactId>commons-logging</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>log4j</groupId> <artifactId>log4j</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-log4j12</artifactId> + <scope>provided</scope> </dependency> </dependencies> http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/9d8b679e/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index e397663..1c5b4c7 100644 --- a/pom.xml +++ b/pom.xml @@ -57,8 +57,28 @@ under the License. <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> <java.source.version>1.6</java.source.version> <java.target.version>1.6</java.target.version> - <scala.version>2.9.2</scala.version> + <!-- + NOTE: Please don't change any dependency versions here without + checking first that they match those on Hadoop's runtime + classpath. + --> + <guava.version>11.0.2</guava.version> + <commons-io.version>2.1</commons-io.version> + <commons-lang.version>2.4</commons-lang.version> + <commons-codec.version>1.4</commons-codec.version> + <commons-httpclient.version>3.0.1</commons-httpclient.version> + <commons-logging.version>1.1.1</commons-logging.version> <avro.version>1.7.0</avro.version> + <jackson.version>1.8.8</jackson.version> + <protobuf-java.version>2.3.0</protobuf-java.version> + <libthrift.version>0.8.0</libthrift.version> + <slf4j.version>1.4.3</slf4j.version> + <log4j.version>1.2.15</log4j.version> + <junit.version>4.10</junit.version> + <hamcrest.version>1.1</hamcrest.version> + <mockito.version>1.9.0</mockito.version> + <scala.version>2.9.2</scala.version> + <scalatest.version>1.7.2</scalatest.version> </properties> <scm> @@ -119,7 +139,7 @@ under the License. <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> - <version>11.0.2</version> + <version>${guava.version}</version> </dependency> <dependency> @@ -137,79 +157,79 @@ under the License. <dependency> <groupId>commons-io</groupId> <artifactId>commons-io</artifactId> - <version>2.2</version> + <version>${commons-io.version}</version> </dependency> <dependency> <groupId>commons-codec</groupId> <artifactId>commons-codec</artifactId> - <version>1.4</version> + <version>${commons-codec.version}</version> </dependency> <dependency> <groupId>commons-lang</groupId> <artifactId>commons-lang</artifactId> - <version>2.4</version> + <version>${commons-lang.version}</version> </dependency> <dependency> <groupId>commons-httpclient</groupId> <artifactId>commons-httpclient</artifactId> - <version>3.0.1</version> + <version>${commons-httpclient.version}</version> </dependency> <dependency> <groupId>org.codehaus.jackson</groupId> <artifactId>jackson-core-asl</artifactId> - <version>1.8.3</version> + <version>${jackson.version}</version> </dependency> <dependency> <groupId>org.codehaus.jackson</groupId> <artifactId>jackson-mapper-asl</artifactId> - <version>1.8.3</version> + <version>${jackson.version}</version> </dependency> <dependency> <groupId>com.google.protobuf</groupId> <artifactId>protobuf-java</artifactId> - <version>2.4.1</version> + <version>${protobuf-java.version}</version> </dependency> <dependency> <groupId>org.apache.thrift</groupId> <artifactId>libthrift</artifactId> - <version>0.8.0</version> + <version>${libthrift.version}</version> </dependency> <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> - <version>4.10</version> + <version>${junit.version}</version> </dependency> <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-all</artifactId> - <version>1.9.0</version> + <version>${mockito.version}</version> </dependency> <dependency> <groupId>org.hamcrest</groupId> <artifactId>hamcrest-all</artifactId> - <version>1.1</version> + <version>${hamcrest.version}</version> </dependency> <dependency> <groupId>commons-logging</groupId> <artifactId>commons-logging</artifactId> - <version>1.1.1</version> + <version>${commons-logging.version}</version> </dependency> <dependency> <groupId>log4j</groupId> <artifactId>log4j</artifactId> - <version>1.2.15</version> + <version>${log4j.version}</version> <exclusions> <exclusion> <groupId>javax.mail</groupId> @@ -233,13 +253,13 @@ under the License. <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> - <version>1.4.3</version> + <version>${slf4j.version}</version> </dependency> <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-log4j12</artifactId> - <version>1.4.3</version> + <version>${slf4j.version}</version> </dependency> <dependency> @@ -263,7 +283,7 @@ under the License. <dependency> <groupId>org.scalatest</groupId> <artifactId>scalatest_${scala.version}</artifactId> - <version>1.7.2</version> + <version>${scalatest.version}</version> </dependency> </dependencies> @@ -357,6 +377,9 @@ under the License. mvn clean install -Dhadoop.profile=2.0 for this to work. --> <hbase.version>0.94.1</hbase.version> + <commons-lang.version>2.5</commons-lang.version> + <protobuf-java.version>2.4.0a</protobuf-java.version> + <slf4j.version>1.6.1</slf4j.version> </properties> <dependencyManagement> <dependencies>
