This is an automated email from the ASF dual-hosted git repository. yangjie01 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 6f79faadc016 [SPARK-44977][SPARK-46257][BUILD][CORE][SQL] Upgrade Derby to 10.16.1.1 6f79faadc016 is described below commit 6f79faadc0166eb357df00fafd84e5bc24558cb4 Author: yangjie01 <yangji...@baidu.com> AuthorDate: Wed Dec 6 11:14:24 2023 +0800 [SPARK-44977][SPARK-46257][BUILD][CORE][SQL] Upgrade Derby to 10.16.1.1 ### What changes were proposed in this pull request? This PR aims to upgrade Derby from 10.14.2.0 to 10.16.1.1: 1. The version in `pom.xml` and related comments were updated. After the update, Spark added a dependency on the `org.apache.derby:derbyshared` module because it is a cascading dependency of `derby`. 2. Since `org.apache.derby.jdbc.EmbeddedDriver` has been moved to the `org.apache.derby:derbytools` module, the corresponding dependency was added to the project. 3. Since `org.apache.derby.jdbc.AutoloadedDriver` has been moved to `org.apache.derby.iapi.jdbc.AutoloadedDriver`, the corresponding test configuration in `o.a.spark.sql.hive.thriftserver.CliSuite` was modified. ### Why are the changes needed? Derby 10.16 release family [supports Java 17](https://issues.apache.org/jira/browse/DERBY-7137) and higher with JDBC 4.2, and this version [no longer supports the Java SecurityManager](https://issues.apache.org/jira/browse/DERBY-7138). The full release notes as follows: - https://db.apache.org/derby/releases/release-10_16_1_1.cgi ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Pass GitHub Actions ### Was this patch authored or co-authored using generative AI tooling? No Closes #44174 from LuciferYang/SPARK-46257. Lead-authored-by: yangjie01 <yangji...@baidu.com> Co-authored-by: YangJie <yangji...@baidu.com> Signed-off-by: yangjie01 <yangji...@baidu.com> --- core/pom.xml | 5 +++++ dev/deps/spark-deps-hadoop-3-hive-2.3 | 4 +++- pom.xml | 9 +++++++-- sql/core/pom.xml | 5 +++++ .../scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala | 2 +- sql/hive/pom.xml | 4 ++++ 6 files changed, 25 insertions(+), 4 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index 4cb0fe055fd0..f2d1f4eb144a 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -347,6 +347,11 @@ <artifactId>derby</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.derby</groupId> + <artifactId>derbytools</artifactId> + <scope>test</scope> + </dependency> <dependency> <groupId>org.seleniumhq.selenium</groupId> <artifactId>selenium-java</artifactId> diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 index ebfe6acad960..ba7c7aee894a 100644 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -59,7 +59,9 @@ datanucleus-core/4.1.17//datanucleus-core-4.1.17.jar datanucleus-rdbms/4.1.19//datanucleus-rdbms-4.1.19.jar datasketches-java/3.3.0//datasketches-java-3.3.0.jar datasketches-memory/2.1.0//datasketches-memory-2.1.0.jar -derby/10.14.2.0//derby-10.14.2.0.jar +derby/10.16.1.1//derby-10.16.1.1.jar +derbyshared/10.16.1.1//derbyshared-10.16.1.1.jar +derbytools/10.16.1.1//derbytools-10.16.1.1.jar dropwizard-metrics-hadoop-metrics2-reporter/0.1.2//dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar flatbuffers-java/1.12.0//flatbuffers-java-1.12.0.jar gcs-connector/hadoop3-2.2.18/shaded/gcs-connector-hadoop3-2.2.18-shaded.jar diff --git a/pom.xml b/pom.xml index 27ee42f103dd..33cb6cf257f6 100644 --- a/pom.xml +++ b/pom.xml @@ -138,8 +138,8 @@ <hive.version.short>2.3</hive.version.short> <!-- note that this should be compatible with Kafka brokers version 0.10 and up --> <kafka.version>3.4.1</kafka.version> - <!-- After 10.15.1.3, the minimum required version is JDK9 --> - <derby.version>10.14.2.0</derby.version> + <!-- After 10.17.1.0, the minimum required version is JDK19 --> + <derby.version>10.16.1.1</derby.version> <parquet.version>1.13.1</parquet.version> <orc.version>1.9.2</orc.version> <orc.classifier>shaded-protobuf</orc.classifier> @@ -962,6 +962,11 @@ <artifactId>derby</artifactId> <version>${derby.version}</version> </dependency> + <dependency> + <groupId>org.apache.derby</groupId> + <artifactId>derbytools</artifactId> + <version>${derby.version}</version> + </dependency> <dependency> <groupId>io.dropwizard.metrics</groupId> <artifactId>metrics-core</artifactId> diff --git a/sql/core/pom.xml b/sql/core/pom.xml index 185d50c018c6..c83104a6bd45 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -194,6 +194,11 @@ <artifactId>derby</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.derby</groupId> + <artifactId>derbytools</artifactId> + <scope>test</scope> + </dependency> <dependency> <groupId>org.apache.parquet</groupId> <artifactId>parquet-avro</artifactId> diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala index 649c985cade9..bbcbf80126b0 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala @@ -812,7 +812,7 @@ class CliSuite extends SparkFunSuite { val catalogUrl = s"spark.sql.catalog.$catalogName.url=jdbc:derby:memory:$catalogName;create=true" val catalogDriver = - s"spark.sql.catalog.$catalogName.driver=org.apache.derby.jdbc.AutoloadedDriver" + s"spark.sql.catalog.$catalogName.driver=org.apache.derby.iapi.jdbc.AutoloadedDriver" val catalogConfigs = Seq(catalogImpl, catalogDriver, catalogUrl, "spark.sql.catalogImplementation=in-memory") .flatMap(Seq("--conf", _)) diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index 8b8842333793..42618fe5bc22 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -172,6 +172,10 @@ <groupId>org.apache.derby</groupId> <artifactId>derby</artifactId> </dependency> + <dependency> + <groupId>org.apache.derby</groupId> + <artifactId>derbytools</artifactId> + </dependency> <dependency> <groupId>org.scala-lang</groupId> <artifactId>scala-compiler</artifactId> --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org