Repository: spark Updated Branches: refs/heads/master 4e3fbe8cd -> c243b21a8
SPARK-3039: Allow spark to be built using avro-mapred for hadoop2 SPARK-3039: Adds the maven property "avro.mapred.classifier" to build spark-assembly with avro-mapred with support for the new Hadoop API. Sets this property to hadoop2 for Hadoop 2 profiles. I am not very familiar with maven, nor do I know whether this potentially breaks something in the hive part of spark. There might be a more elegant way of doing this. Author: Bertrand Bossy <bertrandbo...@gmail.com> Closes #1945 from bbossy/SPARK-3039 and squashes the following commits: c32ce59 [Bertrand Bossy] SPARK-3039: Allow spark to be built using avro-mapred for hadoop2 Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c243b21a Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c243b21a Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c243b21a Branch: refs/heads/master Commit: c243b21a8ba2610266702e00d7d4b5443cb1f687 Parents: 4e3fbe8 Author: Bertrand Bossy <bertrandbo...@gmail.com> Authored: Sun Sep 14 21:10:17 2014 -0700 Committer: Patrick Wendell <pwend...@gmail.com> Committed: Sun Sep 14 21:10:17 2014 -0700 ---------------------------------------------------------------------- pom.xml | 5 +++++ sql/hive/pom.xml | 9 +++++++++ 2 files changed, 14 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/c243b21a/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 2876347..520aed3 100644 --- a/pom.xml +++ b/pom.xml @@ -134,6 +134,7 @@ <chill.version>0.3.6</chill.version> <codahale.metrics.version>3.0.0</codahale.metrics.version> <avro.version>1.7.6</avro.version> + <avro.mapred.classifier></avro.mapred.classifier> <jets3t.version>0.7.1</jets3t.version> <aws.java.sdk.version>1.8.3</aws.java.sdk.version> <aws.kinesis.client.version>1.1.0</aws.kinesis.client.version> @@ -621,6 +622,7 @@ <groupId>org.apache.avro</groupId> <artifactId>avro-mapred</artifactId> <version>${avro.version}</version> + <classifier>${avro.mapred.classifier}</classifier> <exclusions> <exclusion> <groupId>io.netty</groupId> @@ -1108,6 +1110,7 @@ <properties> <hadoop.version>2.2.0</hadoop.version> <protobuf.version>2.5.0</protobuf.version> + <avro.mapred.classifier>hadoop2</avro.mapred.classifier> </properties> </profile> @@ -1117,6 +1120,7 @@ <hadoop.version>2.3.0</hadoop.version> <protobuf.version>2.5.0</protobuf.version> <jets3t.version>0.9.0</jets3t.version> + <avro.mapred.classifier>hadoop2</avro.mapred.classifier> </properties> </profile> @@ -1126,6 +1130,7 @@ <hadoop.version>2.4.0</hadoop.version> <protobuf.version>2.5.0</protobuf.version> <jets3t.version>0.9.0</jets3t.version> + <avro.mapred.classifier>hadoop2</avro.mapred.classifier> </properties> </profile> http://git-wip-us.apache.org/repos/asf/spark/blob/c243b21a/sql/hive/pom.xml ---------------------------------------------------------------------- diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index 45a4c6d..9d7a02b 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -95,6 +95,15 @@ <dependency> <groupId>org.apache.avro</groupId> <artifactId>avro</artifactId> + <version>${avro.version}</version> + </dependency> + <!-- use the build matching the hadoop api of avro-mapred (i.e. no classifier for hadoop 1 API, + hadoop2 classifier for hadoop 2 API. avro-mapred is a dependency of org.spark-project.hive:hive-serde --> + <dependency> + <groupId>org.apache.avro</groupId> + <artifactId>avro-mapred</artifactId> + <version>${avro.version}</version> + <classifier>${avro.mapred.classifier}</classifier> </dependency> <dependency> <groupId>org.scalatest</groupId> --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
