This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 1e9b0f6da96 [SPARK-43836][BUILD] Make Scala 2.13 as default in Spark 3.5 1e9b0f6da96 is described below commit 1e9b0f6da967647f6f9f8125e1cf4b45736d346c Author: Dongjoon Hyun <dongj...@apache.org> AuthorDate: Sun May 28 11:21:52 2023 -0700 [SPARK-43836][BUILD] Make Scala 2.13 as default in Spark 3.5 ### What changes were proposed in this pull request? This PR aims to make Scala 2.13 the default Scala version in Apache Spark 3.5. ### Why are the changes needed? The current releases of Scala community are `Scala 3.2.2` and `Scala 2.13.10`. - https://scala-lang.org/download/all.html Although the Apache Spark community has been using Scala 2.12 by default since Apache Spark 3.0 and Scala community will release Scala 2.12.18 for Java 21 support, we had better focus on `Scala 2.13+` more from Apache Spark 3.5 timeline to adopt Scala community's activity. Since SPARK-25075 added Scala 2.13 at Apache Spark 3.2.0, the Apache Spark community has been using it as a second Scala version. This PR aims to switch only the default Scala version from 2.12 to 2.13. Apache Spark will support both Scala 2.12 and 2.13 still. ### Does this PR introduce _any_ user-facing change? Yes, but we still have Scala 2.12. ### How was this patch tested? Pass the CIs. Closes #41344 from dongjoon-hyun/SPARK-43836. Authored-by: Dongjoon Hyun <dongj...@apache.org> Signed-off-by: Dongjoon Hyun <dongj...@apache.org> --- assembly/pom.xml | 4 +- common/kvstore/pom.xml | 4 +- common/network-common/pom.xml | 4 +- common/network-shuffle/pom.xml | 4 +- common/network-yarn/pom.xml | 4 +- common/sketch/pom.xml | 4 +- common/tags/pom.xml | 4 +- common/unsafe/pom.xml | 4 +- common/utils/pom.xml | 4 +- connector/avro/pom.xml | 8 +- connector/connect/client/jvm/pom.xml | 4 +- connector/connect/common/pom.xml | 4 +- connector/connect/server/pom.xml | 8 +- connector/docker-integration-tests/pom.xml | 4 +- connector/kafka-0-10-assembly/pom.xml | 4 +- connector/kafka-0-10-sql/pom.xml | 8 +- connector/kafka-0-10-token-provider/pom.xml | 4 +- connector/kafka-0-10/pom.xml | 8 +- connector/kinesis-asl-assembly/pom.xml | 4 +- connector/kinesis-asl/pom.xml | 4 +- connector/protobuf/pom.xml | 8 +- connector/spark-ganglia-lgpl/pom.xml | 4 +- core/pom.xml | 8 +- dev/deps/spark-deps-hadoop-3-hive-2.3 | 43 +++---- dev/mima | 8 +- dev/test-dependencies.sh | 2 +- docs/_plugins/copy_api_dirs.rb | 14 +-- examples/pom.xml | 4 +- graphx/pom.xml | 4 +- hadoop-cloud/pom.xml | 4 +- launcher/pom.xml | 4 +- mllib-local/pom.xml | 4 +- mllib/pom.xml | 8 +- pom.xml | 125 ++++++++++----------- repl/pom.xml | 4 +- resource-managers/kubernetes/core/pom.xml | 4 +- .../kubernetes/integration-tests/pom.xml | 4 +- resource-managers/mesos/pom.xml | 4 +- resource-managers/yarn/pom.xml | 4 +- sql/catalyst/pom.xml | 8 +- sql/core/pom.xml | 8 +- sql/hive-thriftserver/pom.xml | 8 +- sql/hive/pom.xml | 8 +- streaming/pom.xml | 8 +- tools/pom.xml | 4 +- 45 files changed, 197 insertions(+), 203 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index 09d6bd8a33f..d4d7a1db4a2 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -20,12 +20,12 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> - <artifactId>spark-assembly_2.12</artifactId> + <artifactId>spark-assembly_2.13</artifactId> <name>Spark Project Assembly</name> <url>https://spark.apache.org/</url> <packaging>pom</packaging> diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml index bef8303874b..69f9f186e08 100644 --- a/common/kvstore/pom.xml +++ b/common/kvstore/pom.xml @@ -21,12 +21,12 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> - <artifactId>spark-kvstore_2.12</artifactId> + <artifactId>spark-kvstore_2.13</artifactId> <packaging>jar</packaging> <name>Spark Project Local DB</name> <url>https://spark.apache.org/</url> diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml index 8a63e999c53..9f90d12216e 100644 --- a/common/network-common/pom.xml +++ b/common/network-common/pom.xml @@ -21,12 +21,12 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> - <artifactId>spark-network-common_2.12</artifactId> + <artifactId>spark-network-common_2.13</artifactId> <packaging>jar</packaging> <name>Spark Project Networking</name> <url>https://spark.apache.org/</url> diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml index a8bde14a259..864f1cc2d37 100644 --- a/common/network-shuffle/pom.xml +++ b/common/network-shuffle/pom.xml @@ -21,12 +21,12 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> - <artifactId>spark-network-shuffle_2.12</artifactId> + <artifactId>spark-network-shuffle_2.13</artifactId> <packaging>jar</packaging> <name>Spark Project Shuffle Streaming Service</name> <url>https://spark.apache.org/</url> diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml index 671d5cb7e01..c19ac33afa5 100644 --- a/common/network-yarn/pom.xml +++ b/common/network-yarn/pom.xml @@ -21,12 +21,12 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> - <artifactId>spark-network-yarn_2.12</artifactId> + <artifactId>spark-network-yarn_2.13</artifactId> <packaging>jar</packaging> <name>Spark Project YARN Shuffle Service</name> <url>https://spark.apache.org/</url> diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml index 4cc597519c3..6cf1a4fb83e 100644 --- a/common/sketch/pom.xml +++ b/common/sketch/pom.xml @@ -21,12 +21,12 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> - <artifactId>spark-sketch_2.12</artifactId> + <artifactId>spark-sketch_2.13</artifactId> <packaging>jar</packaging> <name>Spark Project Sketch</name> <url>https://spark.apache.org/</url> diff --git a/common/tags/pom.xml b/common/tags/pom.xml index 9a44c847d8a..1eb8352e32d 100644 --- a/common/tags/pom.xml +++ b/common/tags/pom.xml @@ -21,12 +21,12 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> - <artifactId>spark-tags_2.12</artifactId> + <artifactId>spark-tags_2.13</artifactId> <packaging>jar</packaging> <name>Spark Project Tags</name> <url>https://spark.apache.org/</url> diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml index a61f00084eb..e37f1b80a50 100644 --- a/common/unsafe/pom.xml +++ b/common/unsafe/pom.xml @@ -21,12 +21,12 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> - <artifactId>spark-unsafe_2.12</artifactId> + <artifactId>spark-unsafe_2.13</artifactId> <packaging>jar</packaging> <name>Spark Project Unsafe</name> <url>https://spark.apache.org/</url> diff --git a/common/utils/pom.xml b/common/utils/pom.xml index 8d24d888e6d..53df20b646d 100644 --- a/common/utils/pom.xml +++ b/common/utils/pom.xml @@ -21,12 +21,12 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> - <artifactId>spark-common-utils_2.12</artifactId> + <artifactId>spark-common-utils_2.13</artifactId> <packaging>jar</packaging> <name>Spark Project Common Utils</name> <url>https://spark.apache.org/</url> diff --git a/connector/avro/pom.xml b/connector/avro/pom.xml index 597e3c2235f..7087fdbccd0 100644 --- a/connector/avro/pom.xml +++ b/connector/avro/pom.xml @@ -20,12 +20,12 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> - <artifactId>spark-avro_2.12</artifactId> + <artifactId>spark-avro_2.13</artifactId> <properties> <sbt.project.name>avro</sbt.project.name> </properties> @@ -70,12 +70,12 @@ <groupId>org.apache.spark</groupId> <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> - <!-- #if scala-2.13 --><!-- + <!-- #if scala-2.13 --> <dependency> <groupId>org.scala-lang.modules</groupId> <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId> </dependency> - --><!-- #endif scala-2.13 --> + <!-- #endif scala-2.13 --> <dependency> <groupId>org.tukaani</groupId> <artifactId>xz</artifactId> diff --git a/connector/connect/client/jvm/pom.xml b/connector/connect/client/jvm/pom.xml index 4d0a4379329..8dc4c3b7a19 100644 --- a/connector/connect/client/jvm/pom.xml +++ b/connector/connect/client/jvm/pom.xml @@ -21,12 +21,12 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../../../../pom.xml</relativePath> </parent> - <artifactId>spark-connect-client-jvm_2.12</artifactId> + <artifactId>spark-connect-client-jvm_2.13</artifactId> <packaging>jar</packaging> <name>Spark Project Connect Client</name> <url>https://spark.apache.org/</url> diff --git a/connector/connect/common/pom.xml b/connector/connect/common/pom.xml index 06076646df7..fcd72480e0e 100644 --- a/connector/connect/common/pom.xml +++ b/connector/connect/common/pom.xml @@ -21,12 +21,12 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../../../pom.xml</relativePath> </parent> - <artifactId>spark-connect-common_2.12</artifactId> + <artifactId>spark-connect-common_2.13</artifactId> <packaging>jar</packaging> <name>Spark Project Connect Common</name> <url>https://spark.apache.org/</url> diff --git a/connector/connect/server/pom.xml b/connector/connect/server/pom.xml index f9dc40ce85a..f3b43e14cda 100644 --- a/connector/connect/server/pom.xml +++ b/connector/connect/server/pom.xml @@ -21,12 +21,12 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../../../pom.xml</relativePath> </parent> - <artifactId>spark-connect_2.12</artifactId> + <artifactId>spark-connect_2.13</artifactId> <packaging>jar</packaging> <name>Spark Project Connect Server</name> <url>https://spark.apache.org/</url> @@ -156,12 +156,12 @@ </exclusion> </exclusions> </dependency> - <!-- #if scala-2.13 --><!-- + <!-- #if scala-2.13 --> <dependency> <groupId>org.scala-lang.modules</groupId> <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId> </dependency> - --><!-- #endif scala-2.13 --> + <!-- #endif scala-2.13 --> <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> diff --git a/connector/docker-integration-tests/pom.xml b/connector/docker-integration-tests/pom.xml index cc549487a8b..e4026983852 100644 --- a/connector/docker-integration-tests/pom.xml +++ b/connector/docker-integration-tests/pom.xml @@ -21,12 +21,12 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> - <artifactId>spark-docker-integration-tests_2.12</artifactId> + <artifactId>spark-docker-integration-tests_2.13</artifactId> <packaging>jar</packaging> <name>Spark Project Docker Integration Tests</name> <url>https://spark.apache.org/</url> diff --git a/connector/kafka-0-10-assembly/pom.xml b/connector/kafka-0-10-assembly/pom.xml index 340974cc789..f339e8c2e4f 100644 --- a/connector/kafka-0-10-assembly/pom.xml +++ b/connector/kafka-0-10-assembly/pom.xml @@ -20,12 +20,12 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> - <artifactId>spark-streaming-kafka-0-10-assembly_2.12</artifactId> + <artifactId>spark-streaming-kafka-0-10-assembly_2.13</artifactId> <packaging>jar</packaging> <name>Spark Integration for Kafka 0.10 Assembly</name> <url>https://spark.apache.org/</url> diff --git a/connector/kafka-0-10-sql/pom.xml b/connector/kafka-0-10-sql/pom.xml index fdd1196cd44..f5a12b61c2b 100644 --- a/connector/kafka-0-10-sql/pom.xml +++ b/connector/kafka-0-10-sql/pom.xml @@ -20,13 +20,13 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-sql-kafka-0-10_2.12</artifactId> + <artifactId>spark-sql-kafka-0-10_2.13</artifactId> <properties> <sbt.project.name>sql-kafka-0-10</sbt.project.name> </properties> @@ -74,12 +74,12 @@ <type>test-jar</type> <scope>test</scope> </dependency> - <!-- #if scala-2.13 --><!-- + <!-- #if scala-2.13 --> <dependency> <groupId>org.scala-lang.modules</groupId> <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId> </dependency> - --><!-- #endif scala-2.13 --> + <!-- #endif scala-2.13 --> <dependency> <groupId>org.apache.kafka</groupId> <artifactId>kafka-clients</artifactId> diff --git a/connector/kafka-0-10-token-provider/pom.xml b/connector/kafka-0-10-token-provider/pom.xml index 3256130c50f..b3c0889d947 100644 --- a/connector/kafka-0-10-token-provider/pom.xml +++ b/connector/kafka-0-10-token-provider/pom.xml @@ -20,13 +20,13 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-token-provider-kafka-0-10_2.12</artifactId> + <artifactId>spark-token-provider-kafka-0-10_2.13</artifactId> <properties> <sbt.project.name>token-provider-kafka-0-10</sbt.project.name> </properties> diff --git a/connector/kafka-0-10/pom.xml b/connector/kafka-0-10/pom.xml index 706eb2dd2c3..f1820bb595a 100644 --- a/connector/kafka-0-10/pom.xml +++ b/connector/kafka-0-10/pom.xml @@ -20,12 +20,12 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> - <artifactId>spark-streaming-kafka-0-10_2.12</artifactId> + <artifactId>spark-streaming-kafka-0-10_2.13</artifactId> <properties> <sbt.project.name>streaming-kafka-0-10</sbt.project.name> </properties> @@ -59,12 +59,12 @@ <type>test-jar</type> <scope>test</scope> </dependency> - <!-- #if scala-2.13 --><!-- + <!-- #if scala-2.13 --> <dependency> <groupId>org.scala-lang.modules</groupId> <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId> </dependency> - --><!-- #endif scala-2.13 --> + <!-- #endif scala-2.13 --> <dependency> <groupId>org.apache.kafka</groupId> <artifactId>kafka-clients</artifactId> diff --git a/connector/kinesis-asl-assembly/pom.xml b/connector/kinesis-asl-assembly/pom.xml index cd5c0393f6f..2cba2668f04 100644 --- a/connector/kinesis-asl-assembly/pom.xml +++ b/connector/kinesis-asl-assembly/pom.xml @@ -20,12 +20,12 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> - <artifactId>spark-streaming-kinesis-asl-assembly_2.12</artifactId> + <artifactId>spark-streaming-kinesis-asl-assembly_2.13</artifactId> <packaging>jar</packaging> <name>Spark Project Kinesis Assembly</name> <url>https://spark.apache.org/</url> diff --git a/connector/kinesis-asl/pom.xml b/connector/kinesis-asl/pom.xml index c70a073e734..af9cd4b7ec9 100644 --- a/connector/kinesis-asl/pom.xml +++ b/connector/kinesis-asl/pom.xml @@ -19,13 +19,13 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> <!-- Kinesis integration is not included by default due to ASL-licensed code. --> - <artifactId>spark-streaming-kinesis-asl_2.12</artifactId> + <artifactId>spark-streaming-kinesis-asl_2.13</artifactId> <packaging>jar</packaging> <name>Spark Kinesis Integration</name> diff --git a/connector/protobuf/pom.xml b/connector/protobuf/pom.xml index e85f07841df..c99b9a06087 100644 --- a/connector/protobuf/pom.xml +++ b/connector/protobuf/pom.xml @@ -20,12 +20,12 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> - <artifactId>spark-protobuf_2.12</artifactId> + <artifactId>spark-protobuf_2.13</artifactId> <properties> <sbt.project.name>protobuf</sbt.project.name> </properties> @@ -70,12 +70,12 @@ <groupId>org.apache.spark</groupId> <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> - <!-- #if scala-2.13 --><!-- + <!-- #if scala-2.13 --> <dependency> <groupId>org.scala-lang.modules</groupId> <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId> </dependency> - --><!-- #endif scala-2.13 --> + <!-- #endif scala-2.13 --> <dependency> <groupId>com.google.protobuf</groupId> <artifactId>protobuf-java</artifactId> diff --git a/connector/spark-ganglia-lgpl/pom.xml b/connector/spark-ganglia-lgpl/pom.xml index c0dcde13558..00f4769fd60 100644 --- a/connector/spark-ganglia-lgpl/pom.xml +++ b/connector/spark-ganglia-lgpl/pom.xml @@ -19,13 +19,13 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> <!-- Ganglia integration is not included by default due to LGPL-licensed code --> - <artifactId>spark-ganglia-lgpl_2.12</artifactId> + <artifactId>spark-ganglia-lgpl_2.13</artifactId> <packaging>jar</packaging> <name>Spark Ganglia Integration</name> diff --git a/core/pom.xml b/core/pom.xml index 09b0a2af96f..4413ba6b634 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -20,12 +20,12 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> - <artifactId>spark-core_2.12</artifactId> + <artifactId>spark-core_2.13</artifactId> <packaging>jar</packaging> <name>Spark Project Core</name> <url>https://spark.apache.org/</url> @@ -35,12 +35,12 @@ </properties> <dependencies> - <!-- #if scala-2.13 --><!-- + <!-- #if scala-2.13 --> <dependency> <groupId>org.scala-lang.modules</groupId> <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId> </dependency> - --><!-- #endif scala-2.13 --> + <!-- #endif scala-2.13 --> <dependency> <groupId>org.apache.avro</groupId> <artifactId>avro</artifactId> diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 index d275c827ef1..bed3be0b086 100644 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -5,7 +5,7 @@ RoaringBitmap/0.9.44//RoaringBitmap-0.9.44.jar ST4/4.0.4//ST4-4.0.4.jar activation/1.1.1//activation-1.1.1.jar aircompressor/0.21//aircompressor-0.21.jar -algebra_2.12/2.0.1//algebra_2.12-2.0.1.jar +algebra_2.13/2.8.0//algebra_2.13-2.8.0.jar aliyun-java-sdk-core/4.5.10//aliyun-java-sdk-core-4.5.10.jar aliyun-java-sdk-kms/2.11.0//aliyun-java-sdk-kms-2.11.0.jar aliyun-java-sdk-ram/3.1.0//aliyun-java-sdk-ram-3.1.0.jar @@ -30,11 +30,11 @@ azure-keyvault-core/1.0.0//azure-keyvault-core-1.0.0.jar azure-storage/7.0.1//azure-storage-7.0.1.jar blas/3.0.3//blas-3.0.3.jar bonecp/0.8.0.RELEASE//bonecp-0.8.0.RELEASE.jar -breeze-macros_2.12/2.1.0//breeze-macros_2.12-2.1.0.jar -breeze_2.12/2.1.0//breeze_2.12-2.1.0.jar -cats-kernel_2.12/2.1.1//cats-kernel_2.12-2.1.1.jar +breeze-macros_2.13/2.1.0//breeze-macros_2.13-2.1.0.jar +breeze_2.13/2.1.0//breeze_2.13-2.1.0.jar +cats-kernel_2.13/2.8.0//cats-kernel_2.13-2.8.0.jar chill-java/0.10.0//chill-java-0.10.0.jar -chill_2.12/0.10.0//chill_2.12-0.10.0.jar +chill_2.13/0.10.0//chill_2.13-0.10.0.jar commons-cli/1.5.0//commons-cli-1.5.0.jar commons-codec/1.15//commons-codec-1.15.jar commons-collections/3.2.2//commons-collections-3.2.2.jar @@ -104,7 +104,7 @@ jackson-databind/2.15.1//jackson-databind-2.15.1.jar jackson-dataformat-cbor/2.15.1//jackson-dataformat-cbor-2.15.1.jar jackson-dataformat-yaml/2.15.1//jackson-dataformat-yaml-2.15.1.jar jackson-datatype-jsr310/2.15.1//jackson-datatype-jsr310-2.15.1.jar -jackson-module-scala_2.12/2.15.1//jackson-module-scala_2.12-2.15.1.jar +jackson-module-scala_2.13/2.15.1//jackson-module-scala_2.13-2.15.1.jar jakarta.annotation-api/1.3.5//jakarta.annotation-api-1.3.5.jar jakarta.inject/2.6.1//jakarta.inject-2.6.1.jar jakarta.servlet-api/4.0.3//jakarta.servlet-api-4.0.3.jar @@ -129,14 +129,16 @@ jettison/1.5.3//jettison-1.5.3.jar jetty-util-ajax/9.4.51.v20230217//jetty-util-ajax-9.4.51.v20230217.jar jetty-util/9.4.51.v20230217//jetty-util-9.4.51.v20230217.jar jline/2.14.6//jline-2.14.6.jar +jline/3.21.0//jline-3.21.0.jar +jna/5.9.0//jna-5.9.0.jar joda-time/2.12.5//joda-time-2.12.5.jar jodd-core/3.5.2//jodd-core-3.5.2.jar jpam/1.1//jpam-1.1.jar json/1.8//json-1.8.jar -json4s-ast_2.12/3.7.0-M11//json4s-ast_2.12-3.7.0-M11.jar -json4s-core_2.12/3.7.0-M11//json4s-core_2.12-3.7.0-M11.jar -json4s-jackson_2.12/3.7.0-M11//json4s-jackson_2.12-3.7.0-M11.jar -json4s-scalap_2.12/3.7.0-M11//json4s-scalap_2.12-3.7.0-M11.jar +json4s-ast_2.13/3.7.0-M11//json4s-ast_2.13-3.7.0-M11.jar +json4s-core_2.13/3.7.0-M11//json4s-core_2.13-3.7.0-M11.jar +json4s-jackson_2.13/3.7.0-M11//json4s-jackson_2.13-3.7.0-M11.jar +json4s-scalap_2.13/3.7.0-M11//json4s-scalap_2.13-3.7.0-M11.jar jsr305/3.0.0//jsr305-3.0.0.jar jta/1.1//jta-1.1.jar jul-to-slf4j/2.0.7//jul-to-slf4j-2.0.7.jar @@ -224,21 +226,22 @@ pickle/1.3//pickle-1.3.jar py4j/0.10.9.7//py4j-0.10.9.7.jar remotetea-oncrpc/1.1.2//remotetea-oncrpc-1.1.2.jar rocksdbjni/8.1.1.1//rocksdbjni-8.1.1.1.jar -scala-collection-compat_2.12/2.7.0//scala-collection-compat_2.12-2.7.0.jar -scala-compiler/2.12.17//scala-compiler-2.12.17.jar -scala-library/2.12.17//scala-library-2.12.17.jar -scala-parser-combinators_2.12/2.2.0//scala-parser-combinators_2.12-2.2.0.jar -scala-reflect/2.12.17//scala-reflect-2.12.17.jar -scala-xml_2.12/2.1.0//scala-xml_2.12-2.1.0.jar +scala-collection-compat_2.13/2.7.0//scala-collection-compat_2.13-2.7.0.jar +scala-compiler/2.13.8//scala-compiler-2.13.8.jar +scala-library/2.13.8//scala-library-2.13.8.jar +scala-parallel-collections_2.13/1.0.4//scala-parallel-collections_2.13-1.0.4.jar +scala-parser-combinators_2.13/2.2.0//scala-parser-combinators_2.13-2.2.0.jar +scala-reflect/2.13.8//scala-reflect-2.13.8.jar +scala-xml_2.13/2.1.0//scala-xml_2.13-2.1.0.jar shims/0.9.44//shims-0.9.44.jar slf4j-api/2.0.7//slf4j-api-2.0.7.jar snakeyaml-engine/2.6//snakeyaml-engine-2.6.jar snakeyaml/2.0//snakeyaml-2.0.jar snappy-java/1.1.10.0//snappy-java-1.1.10.0.jar -spire-macros_2.12/0.17.0//spire-macros_2.12-0.17.0.jar -spire-platform_2.12/0.17.0//spire-platform_2.12-0.17.0.jar -spire-util_2.12/0.17.0//spire-util_2.12-0.17.0.jar -spire_2.12/0.17.0//spire_2.12-0.17.0.jar +spire-macros_2.13/0.18.0//spire-macros_2.13-0.18.0.jar +spire-platform_2.13/0.18.0//spire-platform_2.13-0.18.0.jar +spire-util_2.13/0.18.0//spire-util_2.13-0.18.0.jar +spire_2.13/0.18.0//spire_2.13-0.18.0.jar stax-api/1.0.1//stax-api-1.0.1.jar stream/2.9.6//stream-2.9.6.jar super-csv/2.2.0//super-csv-2.2.0.jar diff --git a/dev/mima b/dev/mima index 4a9e343b0a7..32c3718e4cc 100755 --- a/dev/mima +++ b/dev/mima @@ -24,9 +24,9 @@ set -e FWDIR="$(cd "`dirname "$0"`"/..; pwd)" cd "$FWDIR" -SPARK_PROFILES=${1:-"-Pmesos -Pkubernetes -Pyarn -Pspark-ganglia-lgpl -Pkinesis-asl -Phive-thriftserver -Phive"} -TOOLS_CLASSPATH="$(build/sbt -DcopyDependencies=false "export tools/fullClasspath" | grep jar | tail -n1)" -OLD_DEPS_CLASSPATH="$(build/sbt -DcopyDependencies=false $SPARK_PROFILES "export oldDeps/fullClasspath" | grep jar | tail -n1)" +SPARK_PROFILES=${1:-"-Pscala-2.13 -Pmesos -Pkubernetes -Pyarn -Pspark-ganglia-lgpl -Pkinesis-asl -Phive-thriftserver -Phive"} +TOOLS_CLASSPATH="$(build/sbt -Pscala-2.13 -DcopyDependencies=false "export tools/fullClasspath" | grep jar | tail -n1)" +OLD_DEPS_CLASSPATH="$(build/sbt -Pscala-2.13 -DcopyDependencies=false $SPARK_PROFILES "export oldDeps/fullClasspath" | grep jar | tail -n1)" rm -f .generated-mima* @@ -42,7 +42,7 @@ $JAVA_CMD \ -cp "$TOOLS_CLASSPATH:$OLD_DEPS_CLASSPATH" \ org.apache.spark.tools.GenerateMIMAIgnore -echo -e "q\n" | build/sbt -mem 5120 -DcopyDependencies=false "$@" mimaReportBinaryIssues | grep -v -e "info.*Resolving" +echo -e "q\n" | build/sbt -Pscala-2.13 -mem 5120 -DcopyDependencies=false "$@" mimaReportBinaryIssues | grep -v -e "info.*Resolving" ret_val=$? if [ $ret_val != 0 ]; then diff --git a/dev/test-dependencies.sh b/dev/test-dependencies.sh index d7967ac3afa..07da6497b97 100755 --- a/dev/test-dependencies.sh +++ b/dev/test-dependencies.sh @@ -62,7 +62,7 @@ SCALA_BINARY_VERSION=$($MVN -q \ -Dexec.args='${scala.binary.version}' \ --non-recursive \ org.codehaus.mojo:exec-maven-plugin:1.6.0:exec | grep -E '[0-9]+\.[0-9]+') -if [[ "$SCALA_BINARY_VERSION" != "2.12" ]]; then +if [[ "$SCALA_BINARY_VERSION" != "2.13" ]]; then echo "Skip dependency testing on $SCALA_BINARY_VERSION" exit 0 fi diff --git a/docs/_plugins/copy_api_dirs.rb b/docs/_plugins/copy_api_dirs.rb index 28d5e0d82c9..9cb073ef1e0 100644 --- a/docs/_plugins/copy_api_dirs.rb +++ b/docs/_plugins/copy_api_dirs.rb @@ -26,8 +26,8 @@ if not (ENV['SKIP_API'] == '1') curr_dir = pwd cd("..") - puts "Running 'build/sbt -Pkinesis-asl clean compile unidoc' from " + pwd + "; this may take a few minutes..." - system("build/sbt -Pkinesis-asl clean compile unidoc") || raise("Unidoc generation failed") + puts "Running 'build/sbt -Pscala-2.13 -Pkinesis-asl clean compile unidoc' from " + pwd + "; this may take a few minutes..." + system("build/sbt -Pscala-2.13 -Pkinesis-asl clean compile unidoc") || raise("Unidoc generation failed") puts "Moving back into docs dir." cd("docs") @@ -37,7 +37,7 @@ if not (ENV['SKIP_API'] == '1') # Copy over the unified ScalaDoc for all projects to api/scala. # This directory will be copied over to _site when `jekyll` command is run. - source = "../target/scala-2.12/unidoc" + source = "../target/scala-2.13/unidoc" dest = "api/scala" puts "Making directory " + dest @@ -119,8 +119,8 @@ if not (ENV['SKIP_API'] == '1') puts "Moving to project root and building API docs." cd("..") - puts "Running 'build/sbt clean package -Phive' from " + pwd + "; this may take a few minutes..." - system("build/sbt clean package -Phive") || raise("PySpark doc generation failed") + puts "Running 'build/sbt -Pscala-2.13 clean package -Phive' from " + pwd + "; this may take a few minutes..." + system("build/sbt -Pscala-2.13 clean package -Phive") || raise("PySpark doc generation failed") puts "Moving back into docs dir." cd("docs") @@ -165,8 +165,8 @@ if not (ENV['SKIP_API'] == '1') puts "Moving to project root and building API docs." cd("..") - puts "Running 'build/sbt clean package -Phive' from " + pwd + "; this may take a few minutes..." - system("build/sbt clean package -Phive") || raise("SQL doc generation failed") + puts "Running 'build/sbt -Pscala-2.13 clean package -Phive' from " + pwd + "; this may take a few minutes..." + system("build/sbt -Pscala-2.13 clean package -Phive") || raise("SQL doc generation failed") puts "Moving back into docs dir." cd("docs") diff --git a/examples/pom.xml b/examples/pom.xml index e8f22b995fd..57e41724bdc 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -20,12 +20,12 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> - <artifactId>spark-examples_2.12</artifactId> + <artifactId>spark-examples_2.13</artifactId> <packaging>jar</packaging> <name>Spark Project Examples</name> <url>https://spark.apache.org/</url> diff --git a/graphx/pom.xml b/graphx/pom.xml index 48baeb9a875..5d01dd06c0e 100644 --- a/graphx/pom.xml +++ b/graphx/pom.xml @@ -20,12 +20,12 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> - <artifactId>spark-graphx_2.12</artifactId> + <artifactId>spark-graphx_2.13</artifactId> <properties> <sbt.project.name>graphx</sbt.project.name> </properties> diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml index e213052dbc1..7a3e601553d 100644 --- a/hadoop-cloud/pom.xml +++ b/hadoop-cloud/pom.xml @@ -21,12 +21,12 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> - <artifactId>spark-hadoop-cloud_2.12</artifactId> + <artifactId>spark-hadoop-cloud_2.13</artifactId> <packaging>jar</packaging> <name>Spark Project Hadoop Cloud Integration</name> <description> diff --git a/launcher/pom.xml b/launcher/pom.xml index aba7ee82d53..0bc3ae20ee1 100644 --- a/launcher/pom.xml +++ b/launcher/pom.xml @@ -21,12 +21,12 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> - <artifactId>spark-launcher_2.12</artifactId> + <artifactId>spark-launcher_2.13</artifactId> <packaging>jar</packaging> <name>Spark Project Launcher</name> <url>https://spark.apache.org/</url> diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml index 00c16a8b6a5..83ca643f43b 100644 --- a/mllib-local/pom.xml +++ b/mllib-local/pom.xml @@ -20,12 +20,12 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> - <artifactId>spark-mllib-local_2.12</artifactId> + <artifactId>spark-mllib-local_2.13</artifactId> <properties> <sbt.project.name>mllib-local</sbt.project.name> </properties> diff --git a/mllib/pom.xml b/mllib/pom.xml index 73af83c7586..07290124273 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -20,12 +20,12 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> - <artifactId>spark-mllib_2.12</artifactId> + <artifactId>spark-mllib_2.13</artifactId> <properties> <sbt.project.name>mllib</sbt.project.name> </properties> @@ -91,12 +91,12 @@ <type>test-jar</type> <scope>test</scope> </dependency> - <!-- #if scala-2.13 --><!-- + <!-- #if scala-2.13 --> <dependency> <groupId>org.scala-lang.modules</groupId> <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId> </dependency> - --><!-- #endif scala-2.13 --> + <!-- #endif scala-2.13 --> <dependency> <groupId>org.scalanlp</groupId> <artifactId>breeze_${scala.binary.version}</artifactId> diff --git a/pom.xml b/pom.xml index 2394e429218..cc61c72ed75 100644 --- a/pom.xml +++ b/pom.xml @@ -25,7 +25,7 @@ <version>18</version> </parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <packaging>pom</packaging> <name>Spark Project Parent POM</name> @@ -167,8 +167,8 @@ <!-- managed up from 3.2.1 for SPARK-11652 --> <commons.collections.version>3.2.2</commons.collections.version> <commons.collections4.version>4.4</commons.collections4.version> - <scala.version>2.12.17</scala.version> - <scala.binary.version>2.12</scala.binary.version> + <scala.version>2.13.8</scala.version> + <scala.binary.version>2.13</scala.binary.version> <scalatest-maven-plugin.version>2.2.0</scalatest-maven-plugin.version> <!-- dont update scala-maven-plugin to version 4.8.1 SPARK-42809 and SPARK-43595 --> <scala-maven-plugin.version>4.8.0</scala-maven-plugin.version> @@ -422,13 +422,13 @@ <version>${project.version}</version> <type>test-jar</type> </dependency> - <!-- #if scala-2.13 --><!-- + <!-- #if scala-2.13 --> <dependency> <groupId>org.scala-lang.modules</groupId> <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId> <version>1.0.4</version> </dependency> - --><!-- #endif scala-2.13 --> + <!-- #endif scala-2.13 --> <dependency> <groupId>com.twitter</groupId> <artifactId>chill_${scala.binary.version}</artifactId> @@ -1073,7 +1073,7 @@ <exclusions> <exclusion> <groupId>org.scala-lang.modules</groupId> - <artifactId>scala-xml_2.12</artifactId> + <artifactId>scala-xml_2.13</artifactId> </exclusion> </exclusions> </dependency> @@ -2854,9 +2854,43 @@ <arg>-feature</arg> <arg>-explaintypes</arg> <arg>-target:jvm-1.8</arg> - <arg>-Xfatal-warnings</arg> - <arg>-Ywarn-unused:imports</arg> - <arg>-P:silencer:globalFilters=.*deprecated.*</arg> + <arg>-Wconf:cat=deprecation:wv,any:e</arg> + <!-- + TODO(SPARK-33805): Undo the corresponding deprecated usage suppression rule after fixed + <arg>-Wunused:imports</arg> + --> + <arg>-Wconf:cat=scaladoc:wv</arg> + <arg>-Wconf:cat=lint-multiarg-infix:wv</arg> + <arg>-Wconf:cat=other-nullary-override:wv</arg> + <arg>-Wconf:cat=other-match-analysis&site=org.apache.spark.sql.catalyst.catalog.SessionCatalog.lookupFunction.catalogFunction:wv</arg> + <arg>-Wconf:cat=other-pure-statement&site=org.apache.spark.streaming.util.FileBasedWriteAheadLog.readAll.readFile:wv</arg> + <arg>-Wconf:cat=other-pure-statement&site=org.apache.spark.scheduler.OutputCommitCoordinatorSuite.<local OutputCommitCoordinatorSuite>.futureAction:wv</arg> + <!-- + SPARK-33775 Suppress compilation warnings that contain the following contents. + TODO(SPARK-33805): Undo the corresponding deprecated usage suppression rule after fixed. + --> + <arg>-Wconf:msg=^(?=.*?method|value|type|object|trait|inheritance)(?=.*?deprecated)(?=.*?since 2.13).+$:s</arg> + <arg>-Wconf:msg=^(?=.*?Widening conversion from)(?=.*?is deprecated because it loses precision).+$:s</arg> + <arg>-Wconf:msg=Auto-application to \`\(\)\` is deprecated:s</arg> + <arg>-Wconf:msg=method with a single empty parameter list overrides method without any parameter list:s</arg> + <arg>-Wconf:msg=method without a parameter list overrides a method with a single empty one:s</arg> + <!-- + SPARK-35574 Prevent the recurrence of compilation warnings related to + `procedure syntax is deprecated` + --> + <arg>-Wconf:cat=deprecation&msg=procedure syntax is deprecated:e</arg> + <!-- + SPARK-35496 Upgrade Scala to 2.13.7 and suppress: + 1. `The outer reference in this type test cannot be checked at run time` + 2. `the type test for pattern TypeA cannot be checked at runtime because it + has type parameters eliminated by erasure` + 3. `abstract type TypeA in type pattern Seq[TypeA] (the underlying of + Seq[TypeA]) is unchecked since it is eliminated by erasure` + 4. `fruitless type test: a value of TypeA cannot also be a TypeB` + --> + <arg>-Wconf:cat=unchecked&msg=outer reference:s</arg> + <arg>-Wconf:cat=unchecked&msg=eliminated by erasure:s</arg> + <arg>-Wconf:msg=^(?=.*?a value of type)(?=.*?cannot also be).+$:s</arg> </args> <jvmArgs> <jvmArg>-Xss128m</jvmArg> @@ -2872,13 +2906,6 @@ <javacArg>${java.version}</javacArg> <javacArg>-Xlint:all,-serial,-path,-try</javacArg> </javacArgs> - <compilerPlugins> - <compilerPlugin> - <groupId>com.github.ghik</groupId> - <artifactId>silencer-plugin_${scala.version}</artifactId> - <version>1.7.10</version> - </compilerPlugin> - </compilerPlugins> </configuration> </plugin> <plugin> @@ -3529,6 +3556,13 @@ </properties> </profile> + <profile> + <id>scala-2.13</id> + <properties> + <scala.version>2.13.8</scala.version> + </properties> + </profile> + <profile> <id>scala-2.12</id> <properties> @@ -3538,20 +3572,6 @@ --> <scala.version>2.12.17</scala.version> </properties> - <build> - <pluginManagement> - <plugins> - </plugins> - </pluginManagement> - </build> - </profile> - - <profile> - <id>scala-2.13</id> - <properties> - <scala.version>2.13.8</scala.version> - <scala.binary.version>2.13</scala.binary.version> - </properties> <build> <pluginManagement> <plugins> @@ -3565,45 +3585,16 @@ <arg>-feature</arg> <arg>-explaintypes</arg> <arg>-target:jvm-1.8</arg> - <arg>-Wconf:cat=deprecation:wv,any:e</arg> - <!-- - TODO(SPARK-33805): Undo the corresponding deprecated usage suppression rule after fixed - <arg>-Wunused:imports</arg> - --> - <arg>-Wconf:cat=scaladoc:wv</arg> - <arg>-Wconf:cat=lint-multiarg-infix:wv</arg> - <arg>-Wconf:cat=other-nullary-override:wv</arg> - <arg>-Wconf:cat=other-match-analysis&site=org.apache.spark.sql.catalyst.catalog.SessionCatalog.lookupFunction.catalogFunction:wv</arg> - <arg>-Wconf:cat=other-pure-statement&site=org.apache.spark.streaming.util.FileBasedWriteAheadLog.readAll.readFile:wv</arg> - <arg>-Wconf:cat=other-pure-statement&site=org.apache.spark.scheduler.OutputCommitCoordinatorSuite.<local OutputCommitCoordinatorSuite>.futureAction:wv</arg> - <!-- - SPARK-33775 Suppress compilation warnings that contain the following contents. - TODO(SPARK-33805): Undo the corresponding deprecated usage suppression rule after fixed. - --> - <arg>-Wconf:msg=^(?=.*?method|value|type|object|trait|inheritance)(?=.*?deprecated)(?=.*?since 2.13).+$:s</arg> - <arg>-Wconf:msg=^(?=.*?Widening conversion from)(?=.*?is deprecated because it loses precision).+$:s</arg> - <arg>-Wconf:msg=Auto-application to \`\(\)\` is deprecated:s</arg> - <arg>-Wconf:msg=method with a single empty parameter list overrides method without any parameter list:s</arg> - <arg>-Wconf:msg=method without a parameter list overrides a method with a single empty one:s</arg> - <!-- - SPARK-35574 Prevent the recurrence of compilation warnings related to - `procedure syntax is deprecated` - --> - <arg>-Wconf:cat=deprecation&msg=procedure syntax is deprecated:e</arg> - <!-- - SPARK-35496 Upgrade Scala to 2.13.7 and suppress: - 1. `The outer reference in this type test cannot be checked at run time` - 2. `the type test for pattern TypeA cannot be checked at runtime because it - has type parameters eliminated by erasure` - 3. `abstract type TypeA in type pattern Seq[TypeA] (the underlying of - Seq[TypeA]) is unchecked since it is eliminated by erasure` - 4. `fruitless type test: a value of TypeA cannot also be a TypeB` - --> - <arg>-Wconf:cat=unchecked&msg=outer reference:s</arg> - <arg>-Wconf:cat=unchecked&msg=eliminated by erasure:s</arg> - <arg>-Wconf:msg=^(?=.*?a value of type)(?=.*?cannot also be).+$:s</arg> + <arg>-Xfatal-warnings</arg> + <arg>-Ywarn-unused:imports</arg> + <arg>-P:silencer:globalFilters=.*deprecated.*</arg> </args> <compilerPlugins combine.self="override"> + <compilerPlugin> + <groupId>com.github.ghik</groupId> + <artifactId>silencer-plugin_${scala.version}</artifactId> + <version>1.7.10</version> + </compilerPlugin> </compilerPlugins> </configuration> </plugin> diff --git a/repl/pom.xml b/repl/pom.xml index 8c0f9f989c1..74ac775100c 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -20,12 +20,12 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> - <artifactId>spark-repl_2.12</artifactId> + <artifactId>spark-repl_2.13</artifactId> <packaging>jar</packaging> <name>Spark Project REPL</name> <url>https://spark.apache.org/</url> diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml index 9dab5496184..72c7f1f12f4 100644 --- a/resource-managers/kubernetes/core/pom.xml +++ b/resource-managers/kubernetes/core/pom.xml @@ -19,12 +19,12 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../../../pom.xml</relativePath> </parent> - <artifactId>spark-kubernetes_2.12</artifactId> + <artifactId>spark-kubernetes_2.13</artifactId> <packaging>jar</packaging> <name>Spark Project Kubernetes</name> <properties> diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml index 02894f82eec..3e25e705370 100644 --- a/resource-managers/kubernetes/integration-tests/pom.xml +++ b/resource-managers/kubernetes/integration-tests/pom.xml @@ -19,12 +19,12 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../../../pom.xml</relativePath> </parent> - <artifactId>spark-kubernetes-integration-tests_2.12</artifactId> + <artifactId>spark-kubernetes-integration-tests_2.13</artifactId> <properties> <sbt.project.name>kubernetes-integration-tests</sbt.project.name> diff --git a/resource-managers/mesos/pom.xml b/resource-managers/mesos/pom.xml index 7510ecac3e7..267d6c7d84f 100644 --- a/resource-managers/mesos/pom.xml +++ b/resource-managers/mesos/pom.xml @@ -19,12 +19,12 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> - <artifactId>spark-mesos_2.12</artifactId> + <artifactId>spark-mesos_2.13</artifactId> <packaging>jar</packaging> <name>Spark Project Mesos</name> <properties> diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml index dcc7bcdd1af..2cda552a9c4 100644 --- a/resource-managers/yarn/pom.xml +++ b/resource-managers/yarn/pom.xml @@ -19,12 +19,12 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> - <artifactId>spark-yarn_2.12</artifactId> + <artifactId>spark-yarn_2.13</artifactId> <packaging>jar</packaging> <name>Spark Project YARN</name> <properties> diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml index 9057cc8a20e..cf01c0af4a4 100644 --- a/sql/catalyst/pom.xml +++ b/sql/catalyst/pom.xml @@ -21,12 +21,12 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> - <artifactId>spark-catalyst_2.12</artifactId> + <artifactId>spark-catalyst_2.13</artifactId> <packaging>jar</packaging> <name>Spark Project Catalyst</name> <url>https://spark.apache.org/</url> @@ -87,12 +87,12 @@ <artifactId>spark-sketch_${scala.binary.version}</artifactId> <version>${project.version}</version> </dependency> - <!-- #if scala-2.13 --><!-- + <!-- #if scala-2.13 --> <dependency> <groupId>org.scala-lang.modules</groupId> <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId> </dependency> - --><!-- #endif scala-2.13 --> + <!-- #endif scala-2.13 --> <dependency> <groupId>org.scalacheck</groupId> <artifactId>scalacheck_${scala.binary.version}</artifactId> diff --git a/sql/core/pom.xml b/sql/core/pom.xml index 7f4c2a4cfa5..5d4f7572d00 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -21,12 +21,12 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> - <artifactId>spark-sql_2.12</artifactId> + <artifactId>spark-sql_2.13</artifactId> <packaging>jar</packaging> <name>Spark Project SQL</name> <url>https://spark.apache.org/</url> @@ -89,12 +89,12 @@ <scope>test</scope> </dependency> - <!-- #if scala-2.13 --><!-- + <!-- #if scala-2.13 --> <dependency> <groupId>org.scala-lang.modules</groupId> <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId> </dependency> - --><!-- #endif scala-2.13 --> + <!-- #endif scala-2.13 --> <dependency> <groupId>org.apache.orc</groupId> <artifactId>orc-core</artifactId> diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml index ad7fc0d2ac4..4bbb92d1376 100644 --- a/sql/hive-thriftserver/pom.xml +++ b/sql/hive-thriftserver/pom.xml @@ -21,12 +21,12 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> - <artifactId>spark-hive-thriftserver_2.12</artifactId> + <artifactId>spark-hive-thriftserver_2.13</artifactId> <packaging>jar</packaging> <name>Spark Project Hive Thrift Server</name> <url>https://spark.apache.org/</url> @@ -61,12 +61,12 @@ <type>test-jar</type> <scope>test</scope> </dependency> - <!-- #if scala-2.13 --><!-- + <!-- #if scala-2.13 --> <dependency> <groupId>org.scala-lang.modules</groupId> <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId> </dependency> - --><!-- #endif scala-2.13 --> + <!-- #endif scala-2.13 --> <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index 16d915c233e..b267830a3ad 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -21,12 +21,12 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> - <artifactId>spark-hive_2.12</artifactId> + <artifactId>spark-hive_2.13</artifactId> <packaging>jar</packaging> <name>Spark Project Hive</name> <url>https://spark.apache.org/</url> @@ -79,12 +79,12 @@ <type>test-jar</type> <scope>test</scope> </dependency> - <!-- #if scala-2.13 --><!-- + <!-- #if scala-2.13 --> <dependency> <groupId>org.scala-lang.modules</groupId> <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId> </dependency> - --><!-- #endif scala-2.13 --> + <!-- #endif scala-2.13 --> <dependency> <groupId>${hive.group}</groupId> <artifactId>hive-common</artifactId> diff --git a/streaming/pom.xml b/streaming/pom.xml index bebfd3abcce..a36370a1e8b 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -20,12 +20,12 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> - <artifactId>spark-streaming_2.12</artifactId> + <artifactId>spark-streaming_2.13</artifactId> <properties> <sbt.project.name>streaming</sbt.project.name> </properties> @@ -50,12 +50,12 @@ <groupId>org.apache.spark</groupId> <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> - <!-- #if scala-2.13 --><!-- + <!-- #if scala-2.13 --> <dependency> <groupId>org.scala-lang.modules</groupId> <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId> </dependency> - --><!-- #endif scala-2.13 --> + <!-- #endif scala-2.13 --> <!-- This spark-tags test-dep is needed even though it isn't used in this module, otherwise testing-cmds that exclude diff --git a/tools/pom.xml b/tools/pom.xml index 9dc136d4268..9a088e84bc0 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -19,12 +19,12 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.12</artifactId> + <artifactId>spark-parent_2.13</artifactId> <version>3.5.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> - <artifactId>spark-tools_2.12</artifactId> + <artifactId>spark-tools_2.13</artifactId> <properties> <sbt.project.name>tools</sbt.project.name> </properties> --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org