This is an automated email from the ASF dual-hosted git repository.
srowen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 1a62e6a [SPARK-36712][BUILD] Make scala-parallel-collections in 2.13
POM a direct dependency (not in maven profile)
1a62e6a is described below
commit 1a62e6a2c119df707f15101b03ecff0c3dee62f5
Author: Lukas Rytz <lukas.r...@gmail.com>
AuthorDate: Mon Sep 13 11:06:50 2021 -0500
[SPARK-36712][BUILD] Make scala-parallel-collections in 2.13 POM a direct
dependency (not in maven profile)
As [reported on
`devspark.apache.org`](https://lists.apache.org/thread.html/r84cff66217de438f1389899e6d6891b573780159cd45463acf3657aa%40%3Cdev.spark.apache.org%3E),
the published POMs when building with Scala 2.13 have the
`scala-parallel-collections` dependency only in the `scala-2.13` profile of the
pom.
### What changes were proposed in this pull request?
This PR suggests to work around this by un-commenting the
`scala-parallel-collections` dependency when switching to 2.13 using the the
`change-scala-version.sh` script.
I included an upgrade to scala-parallel-collections version 1.0.3, the
changes compared to 0.2.0 are minor.
- removed OSGi metadata
- renamed some internal inner classes
- added `Automatic-Module-Name`
### Why are the changes needed?
According to the posts, this solves issues for developers that write unit
tests for their applications.
Stephen Coy suggested to use the
https://www.mojohaus.org/flatten-maven-plugin. While this sounds like a more
principled solution, it is possibly too risky to do at this specific point in
time?
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Locally
Closes #33948 from lrytz/parCollDep.
Authored-by: Lukas Rytz <lukas.r...@gmail.com>
Signed-off-by: Sean Owen <sro...@gmail.com>
---
core/pom.xml | 15 ++++++---------
dev/change-scala-version.sh | 12 ++++++++----
external/avro/pom.xml | 17 ++++++-----------
external/kafka-0-10-sql/pom.xml | 17 ++++++-----------
external/kafka-0-10/pom.xml | 18 ++++++------------
mllib/pom.xml | 18 ++++++------------
pom.xml | 16 +++++++---------
sql/catalyst/pom.xml | 18 ++++++------------
sql/core/pom.xml | 15 ++++++---------
sql/hive-thriftserver/pom.xml | 17 ++++++-----------
sql/hive/pom.xml | 15 ++++++---------
streaming/pom.xml | 17 ++++++-----------
12 files changed, 75 insertions(+), 120 deletions(-)
diff --git a/core/pom.xml b/core/pom.xml
index be44964..dbde22f 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -35,6 +35,12 @@
</properties>
<dependencies>
+ <!-- #if scala-2.13 --><!--
+ <dependency>
+ <groupId>org.scala-lang.modules</groupId>
+
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
+ </dependency>
+ --><!-- #endif scala-2.13 -->
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
@@ -639,15 +645,6 @@
</plugins>
</build>
</profile>
- <profile>
- <id>scala-2.13</id>
- <dependencies>
- <dependency>
- <groupId>org.scala-lang.modules</groupId>
-
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
- </dependency>
- </dependencies>
- </profile>
</profiles>
</project>
diff --git a/dev/change-scala-version.sh b/dev/change-scala-version.sh
index 48b7f64..e17a224 100755
--- a/dev/change-scala-version.sh
+++ b/dev/change-scala-version.sh
@@ -54,11 +54,15 @@ sed_i() {
sed -e "$1" "$2" > "$2.tmp" && mv "$2.tmp" "$2"
}
-export -f sed_i
-
BASEDIR=$(dirname $0)/..
-find "$BASEDIR" -name 'pom.xml' -not -path '*target*' -print \
- -exec bash -c "sed_i 's/\(artifactId.*\)_'$FROM_VERSION'/\1_'$TO_VERSION'/g'
{}" \;
+for f in $(find "$BASEDIR" -name 'pom.xml' -not -path '*target*'); do
+ echo $f
+ sed_i 's/\(artifactId.*\)_'$FROM_VERSION'/\1_'$TO_VERSION'/g' $f
+ sed_i 's/^\([[:space:]]*<!-- #if scala-'$TO_VERSION' -->\)<!--/\1/' $f
+ sed_i 's/^\([[:space:]]*\)-->\(<!-- #endif scala-'$TO_VERSION' -->\)/\1\2/'
$f
+ sed_i 's/^\([[:space:]]*<!-- #if scala-'$FROM_VERSION' -->\)/\1<!--/' $f
+ sed_i 's/^\([[:space:]]*\)\(<!-- #endif scala-'$FROM_VERSION'
-->\)/\1-->\2/' $f
+done
# dependency:get is workaround for SPARK-34762 to download the JAR file of
commons-cli.
# Without this, build with Scala 2.13 using SBT will fail because the help
plugin used below downloads only the POM file.
diff --git a/external/avro/pom.xml b/external/avro/pom.xml
index d9d3583..7e414be 100644
--- a/external/avro/pom.xml
+++ b/external/avro/pom.xml
@@ -70,22 +70,17 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-tags_${scala.binary.version}</artifactId>
</dependency>
+ <!-- #if scala-2.13 --><!--
+ <dependency>
+ <groupId>org.scala-lang.modules</groupId>
+
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
+ </dependency>
+ --><!-- #endif scala-2.13 -->
<dependency>
<groupId>org.tukaani</groupId>
<artifactId>xz</artifactId>
</dependency>
</dependencies>
- <profiles>
- <profile>
- <id>scala-2.13</id>
- <dependencies>
- <dependency>
- <groupId>org.scala-lang.modules</groupId>
-
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
- </dependency>
- </dependencies>
- </profile>
- </profiles>
<build>
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
diff --git a/external/kafka-0-10-sql/pom.xml b/external/kafka-0-10-sql/pom.xml
index 8d505cf..7bedcee 100644
--- a/external/kafka-0-10-sql/pom.xml
+++ b/external/kafka-0-10-sql/pom.xml
@@ -74,6 +74,12 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <!-- #if scala-2.13 --><!--
+ <dependency>
+ <groupId>org.scala-lang.modules</groupId>
+
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
+ </dependency>
+ --><!-- #endif scala-2.13 -->
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
@@ -169,17 +175,6 @@
</dependency>
</dependencies>
- <profiles>
- <profile>
- <id>scala-2.13</id>
- <dependencies>
- <dependency>
- <groupId>org.scala-lang.modules</groupId>
-
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
- </dependency>
- </dependencies>
- </profile>
- </profiles>
<build>
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
diff --git a/external/kafka-0-10/pom.xml b/external/kafka-0-10/pom.xml
index 49f1bef..b689f2b 100644
--- a/external/kafka-0-10/pom.xml
+++ b/external/kafka-0-10/pom.xml
@@ -59,6 +59,12 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <!-- #if scala-2.13 --><!--
+ <dependency>
+ <groupId>org.scala-lang.modules</groupId>
+
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
+ </dependency>
+ --><!-- #endif scala-2.13 -->
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
@@ -136,18 +142,6 @@
</dependencies>
- <profiles>
- <profile>
- <id>scala-2.13</id>
- <dependencies>
- <dependency>
- <groupId>org.scala-lang.modules</groupId>
-
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
- </dependency>
- </dependencies>
- </profile>
- </profiles>
-
<build>
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
diff --git a/mllib/pom.xml b/mllib/pom.xml
index e588961..476226d 100644
--- a/mllib/pom.xml
+++ b/mllib/pom.xml
@@ -91,6 +91,12 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <!-- #if scala-2.13 --><!--
+ <dependency>
+ <groupId>org.scala-lang.modules</groupId>
+
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
+ </dependency>
+ --><!-- #endif scala-2.13 -->
<dependency>
<groupId>org.scalanlp</groupId>
<artifactId>breeze_${scala.binary.version}</artifactId>
@@ -157,18 +163,6 @@
</dependencies>
- <profiles>
- <profile>
- <id>scala-2.13</id>
- <dependencies>
- <dependency>
- <groupId>org.scala-lang.modules</groupId>
-
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
- </dependency>
- </dependencies>
- </profile>
- </profiles>
-
<build>
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
diff --git a/pom.xml b/pom.xml
index 87a2758..c5cb837 100644
--- a/pom.xml
+++ b/pom.xml
@@ -399,6 +399,13 @@
<version>${project.version}</version>
<type>test-jar</type>
</dependency>
+ <!-- #if scala-2.13 --><!--
+ <dependency>
+ <groupId>org.scala-lang.modules</groupId>
+
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
+ <version>1.0.3</version>
+ </dependency>
+ --><!-- #endif scala-2.13 -->
<dependency>
<groupId>com.twitter</groupId>
<artifactId>chill_${scala.binary.version}</artifactId>
@@ -3370,15 +3377,6 @@
<scala.version>2.13.5</scala.version>
<scala.binary.version>2.13</scala.binary.version>
</properties>
- <dependencyManagement>
- <dependencies>
- <dependency>
- <groupId>org.scala-lang.modules</groupId>
-
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
- <version>0.2.0</version>
- </dependency>
- </dependencies>
- </dependencyManagement>
<build>
<pluginManagement>
<plugins>
diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml
index c6612c4..32281b5 100644
--- a/sql/catalyst/pom.xml
+++ b/sql/catalyst/pom.xml
@@ -87,6 +87,12 @@
<artifactId>spark-sketch_${scala.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
+ <!-- #if scala-2.13 --><!--
+ <dependency>
+ <groupId>org.scala-lang.modules</groupId>
+
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
+ </dependency>
+ --><!-- #endif scala-2.13 -->
<dependency>
<groupId>org.scalacheck</groupId>
<artifactId>scalacheck_${scala.binary.version}</artifactId>
@@ -191,16 +197,4 @@
</plugin>
</plugins>
</build>
-
- <profiles>
- <profile>
- <id>scala-2.13</id>
- <dependencies>
- <dependency>
- <groupId>org.scala-lang.modules</groupId>
-
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
- </dependency>
- </dependencies>
- </profile>
- </profiles>
</project>
diff --git a/sql/core/pom.xml b/sql/core/pom.xml
index 27260ce..f9c12d1 100644
--- a/sql/core/pom.xml
+++ b/sql/core/pom.xml
@@ -90,6 +90,12 @@
<scope>test</scope>
</dependency>
+ <!-- #if scala-2.13 --><!--
+ <dependency>
+ <groupId>org.scala-lang.modules</groupId>
+
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
+ </dependency>
+ --><!-- #endif scala-2.13 -->
<dependency>
<groupId>org.apache.orc</groupId>
<artifactId>orc-core</artifactId>
@@ -253,15 +259,6 @@
<profiles>
<profile>
- <id>scala-2.13</id>
- <dependencies>
- <dependency>
- <groupId>org.scala-lang.modules</groupId>
-
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
- </dependency>
- </dependencies>
- </profile>
- <profile>
<id>hadoop-2.7</id>
<dependencies>
<dependency>
diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml
index e14017e..764712f 100644
--- a/sql/hive-thriftserver/pom.xml
+++ b/sql/hive-thriftserver/pom.xml
@@ -61,6 +61,12 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <!-- #if scala-2.13 --><!--
+ <dependency>
+ <groupId>org.scala-lang.modules</groupId>
+
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
+ </dependency>
+ --><!-- #endif scala-2.13 -->
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
@@ -138,17 +144,6 @@
<artifactId>commons-cli</artifactId>
</dependency>
</dependencies>
- <profiles>
- <profile>
- <id>scala-2.13</id>
- <dependencies>
- <dependency>
- <groupId>org.scala-lang.modules</groupId>
-
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
- </dependency>
- </dependencies>
- </profile>
- </profiles>
<build>
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml
index 526df64..86cd1ed 100644
--- a/sql/hive/pom.xml
+++ b/sql/hive/pom.xml
@@ -79,6 +79,12 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <!-- #if scala-2.13 --><!--
+ <dependency>
+ <groupId>org.scala-lang.modules</groupId>
+
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
+ </dependency>
+ --><!-- #endif scala-2.13 -->
<!--
<dependency>
<groupId>com.google.guava</groupId>
@@ -217,15 +223,6 @@
</plugins>
</build>
</profile>
- <profile>
- <id>scala-2.13</id>
- <dependencies>
- <dependency>
- <groupId>org.scala-lang.modules</groupId>
-
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
- </dependency>
- </dependencies>
- </profile>
</profiles>
<build>
diff --git a/streaming/pom.xml b/streaming/pom.xml
index 9a98aba..ea14151 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -50,6 +50,12 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-tags_${scala.binary.version}</artifactId>
</dependency>
+ <!-- #if scala-2.13 --><!--
+ <dependency>
+ <groupId>org.scala-lang.modules</groupId>
+
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
+ </dependency>
+ --><!-- #endif scala-2.13 -->
<!--
This spark-tags test-dep is needed even though it isn't used in this
module, otherwise testing-cmds that exclude
@@ -132,15 +138,4 @@
</plugins>
</build>
- <profiles>
- <profile>
- <id>scala-2.13</id>
- <dependencies>
- <dependency>
- <groupId>org.scala-lang.modules</groupId>
-
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
- </dependency>
- </dependencies>
- </profile>
- </profiles>
</project>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
