This is an automated email from the ASF dual-hosted git repository.
jiayu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-sedona.git
The following commit(s) were added to refs/heads/master by this push:
new 2ae9c1c0 [SEDONA-135] Consolidate and upgrade Hadoop version (#653)
2ae9c1c0 is described below
commit 2ae9c1c0f316968b5bc59dc7b15aaaabf6518561
Author: Kengo Seki <[email protected]>
AuthorDate: Wed Aug 3 12:29:24 2022 +0900
[SEDONA-135] Consolidate and upgrade Hadoop version (#653)
---
.github/workflows/python.yml | 2 +-
core/pom.xml | 8 +++++++-
pom.xml | 46 +++++++++++++++++---------------------------
3 files changed, 26 insertions(+), 30 deletions(-)
diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
index 52d23b78..2e4968e2 100644
--- a/.github/workflows/python.yml
+++ b/.github/workflows/python.yml
@@ -87,4 +87,4 @@ jobs:
- env:
SPARK_VERSION: ${{ matrix.spark }}
HADOOP_VERSION: ${{ matrix.hadoop }}
- run: (export
SPARK_HOME=$PWD/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION};export
PYTHONPATH=$SPARK_HOME/python;cd python;pipenv run pytest tests)
\ No newline at end of file
+ run: (export
SPARK_HOME=$PWD/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION};export
PYTHONPATH=$SPARK_HOME/python;cd python;pipenv run pytest tests)
diff --git a/core/pom.xml b/core/pom.xml
index c02508f3..64060cc3 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -41,7 +41,13 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
- <version>2.7.4</version>
+ <version>${hadoop.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>1.8.5</version>
<scope>test</scope>
</dependency>
<dependency>
diff --git a/pom.xml b/pom.xml
index 958c346c..1c1a2434 100644
--- a/pom.xml
+++ b/pom.xml
@@ -100,6 +100,22 @@
<version>${jts.version}</version>
<scope>${dependency.scope}</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>${dependency.scope}</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<dependency>
<groupId>org.datasyslab</groupId>
<artifactId>sernetcdf</artifactId>
@@ -184,34 +200,6 @@
<version>${spark.version}</version>
<scope>${dependency.scope}</scope>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- <version>2.8.2</version>
- <scope>${dependency.scope}</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>2.8.2</version>
- <scope>${dependency.scope}</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <version>2.8.2</version>
- <scope>${dependency.scope}</scope>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
<dependency>
<groupId>org.scala-lang.modules</groupId>
<artifactId>scala-collection-compat_${scala.compat.version}</artifactId>
@@ -533,6 +521,7 @@
<spark.compat.version>3.0</spark.compat.version>
<spark.converter.version>spark3</spark.converter.version>
<sedona.jackson.version>2.13.3</sedona.jackson.version>
+ <hadoop.version>3.2.4</hadoop.version>
<maven.deploy.skip>false</maven.deploy.skip>
</properties>
</profile>
@@ -550,6 +539,7 @@
<spark.compat.version>2.4</spark.compat.version>
<spark.converter.version>spark2</spark.converter.version>
<sedona.jackson.version>2.6.7</sedona.jackson.version>
+ <hadoop.version>2.10.2</hadoop.version>
<maven.deploy.skip>true</maven.deploy.skip>
</properties>
</profile>
