This is an automated email from the ASF dual-hosted git repository.
yuanzhou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new be80045bca [GLUTEN-8852] Fix code compatibility issues for two tests
on Spark-4.0 (#10487)
be80045bca is described below
commit be80045bcab3c5f7793af6f0762e8117cb9e64f6
Author: PHILO-HE <[email protected]>
AuthorDate: Wed Aug 20 21:28:44 2025 +0800
[GLUTEN-8852] Fix code compatibility issues for two tests on Spark-4.0
(#10487)
* Initial
* Fix
* Fix
* Re-include tests
* Fix instantiation SparkSession
---
.github/workflows/velox_backend_x86.yml | 2 --
.../spark/sql/execution/GlutenHiveUDFSuite.scala | 4 +++-
.../execution/VeloxParquetWriteForHiveSuite.scala | 4 +++-
pom.xml | 19 ---------------
.../apache/spark/sql/classic/ClassicTypes.scala | 28 ++++++++++++++++++++++
.../apache/spark/sql/classic/ClassicTypes.scala | 28 ++++++++++++++++++++++
.../apache/spark/sql/classic/ClassicTypes.scala | 28 ++++++++++++++++++++++
.../apache/spark/sql/classic/ClassicTypes.scala | 28 ++++++++++++++++++++++
.../apache/spark/sql/classic/ClassicTypes.scala | 28 ++++++++++++++++++++++
9 files changed, 146 insertions(+), 23 deletions(-)
diff --git a/.github/workflows/velox_backend_x86.yml
b/.github/workflows/velox_backend_x86.yml
index 67ac1f39b2..4cb9ac3a69 100644
--- a/.github/workflows/velox_backend_x86.yml
+++ b/.github/workflows/velox_backend_x86.yml
@@ -240,8 +240,6 @@ jobs:
echo "JAVA_HOME: $JAVA_HOME"
cd $GITHUB_WORKSPACE/
if [ "${{ matrix.spark }}" = "spark-4.0" ]; then
- rm -f
backends-velox/src/test/scala/org/apache/spark/sql/execution/GlutenHiveUDFSuite.scala
\
-
backends-velox/src/test/scala/org/apache/spark/sql/execution/VeloxParquetWriteForHiveSuite.scala
$MVN_CMD clean install -P${{ matrix.spark }} -P${{ matrix.java }}
-Pscala-2.13 -Pbackends-velox -DskipTests
else
$MVN_CMD clean install -P${{ matrix.spark }} -P${{ matrix.java }}
-Pbackends-velox -DskipTests
diff --git
a/backends-velox/src/test/scala/org/apache/spark/sql/execution/GlutenHiveUDFSuite.scala
b/backends-velox/src/test/scala/org/apache/spark/sql/execution/GlutenHiveUDFSuite.scala
index e6505decb6..0a2cdb5b71 100644
---
a/backends-velox/src/test/scala/org/apache/spark/sql/execution/GlutenHiveUDFSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/spark/sql/execution/GlutenHiveUDFSuite.scala
@@ -27,6 +27,7 @@ import org.apache.spark.internal.config.UI.UI_ENABLED
import org.apache.spark.sql.{DataFrame, GlutenQueryTest, Row, SparkSession}
import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode
import org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation
+import org.apache.spark.sql.classic.ClassicTypes._
import org.apache.spark.sql.hive.HiveUtils
import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
import org.apache.spark.sql.test.SQLTestUtils
@@ -43,6 +44,7 @@ class GlutenHiveUDFSuite extends GlutenQueryTest with
SQLTestUtils {
super.beforeAll()
if (_spark == null) {
+ // By default, the classic SparkSession is constructed.
_spark =
SparkSession.builder().config(sparkConf).enableHiveSupport().getOrCreate()
}
@@ -55,7 +57,7 @@ class GlutenHiveUDFSuite extends GlutenQueryTest with
SQLTestUtils {
super.afterAll()
}
- override protected def spark: SparkSession = _spark
+ override protected def spark: ClassicSparkSession =
_spark.asInstanceOf[ClassicSparkSession]
protected def defaultSparkConf: SparkConf = {
val conf = new SparkConf()
diff --git
a/backends-velox/src/test/scala/org/apache/spark/sql/execution/VeloxParquetWriteForHiveSuite.scala
b/backends-velox/src/test/scala/org/apache/spark/sql/execution/VeloxParquetWriteForHiveSuite.scala
index 73040069cf..60ea00be6a 100644
---
a/backends-velox/src/test/scala/org/apache/spark/sql/execution/VeloxParquetWriteForHiveSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/spark/sql/execution/VeloxParquetWriteForHiveSuite.scala
@@ -26,6 +26,7 @@ import org.apache.spark.sql.{GlutenQueryTest, Row,
SparkSession}
import org.apache.spark.sql.SaveMode
import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode
import org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation
+import org.apache.spark.sql.classic.ClassicTypes._
import org.apache.spark.sql.hive.HiveUtils
import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
import org.apache.spark.sql.test.SQLTestUtils
@@ -49,13 +50,14 @@ class VeloxParquetWriteForHiveSuite
super.beforeAll()
if (_spark == null) {
+ // By default, the classic SparkSession is constructed.
_spark =
SparkSession.builder().config(sparkConf).enableHiveSupport().getOrCreate()
}
_spark.sparkContext.setLogLevel("warn")
}
- override protected def spark: SparkSession = _spark
+ override protected def spark: ClassicSparkSession =
_spark.asInstanceOf[ClassicSparkSession]
protected def defaultSparkConf: SparkConf = {
val conf = new SparkConf()
diff --git a/pom.xml b/pom.xml
index c93bad4a42..90cef0a2de 100644
--- a/pom.xml
+++ b/pom.xml
@@ -487,25 +487,6 @@
</execution>
</executions>
</plugin>
- <plugin>
- <groupId>net.alchim31.maven</groupId>
- <artifactId>scala-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>scala-test-compile-first</id>
- <phase>process-test-resources</phase>
- <goals>
- <goal>testCompile</goal>
- </goals>
- <configuration>
- <excludes>
- <exclude>**/GlutenHiveUDFSuite.scala</exclude>
- <exclude>**/VeloxParquetWriteForHiveSuite.scala</exclude>
- </excludes>
- </configuration>
- </execution>
- </executions>
- </plugin>
</plugins>
</build>
</profile>
diff --git
a/shims/spark32/src/main/scala/org/apache/spark/sql/classic/ClassicTypes.scala
b/shims/spark32/src/main/scala/org/apache/spark/sql/classic/ClassicTypes.scala
new file mode 100644
index 0000000000..7235c0b9b9
--- /dev/null
+++
b/shims/spark32/src/main/scala/org/apache/spark/sql/classic/ClassicTypes.scala
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.classic
+
+import org.apache.spark.sql
+
+/**
+ * Prior to Spark 4.0, `ClassicSparkSession` refers to `sql.SparkSession`.
Since Spark 4.0, it
+ * refers to `sql.classic.SparkSession`.
+ */
+object ClassicTypes {
+
+ type ClassicSparkSession = sql.SparkSession
+}
diff --git
a/shims/spark33/src/main/scala/org/apache/spark/sql/classic/ClassicTypes.scala
b/shims/spark33/src/main/scala/org/apache/spark/sql/classic/ClassicTypes.scala
new file mode 100644
index 0000000000..7235c0b9b9
--- /dev/null
+++
b/shims/spark33/src/main/scala/org/apache/spark/sql/classic/ClassicTypes.scala
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.classic
+
+import org.apache.spark.sql
+
+/**
+ * Prior to Spark 4.0, `ClassicSparkSession` refers to `sql.SparkSession`.
Since Spark 4.0, it
+ * refers to `sql.classic.SparkSession`.
+ */
+object ClassicTypes {
+
+ type ClassicSparkSession = sql.SparkSession
+}
diff --git
a/shims/spark34/src/main/scala/org/apache/spark/sql/classic/ClassicTypes.scala
b/shims/spark34/src/main/scala/org/apache/spark/sql/classic/ClassicTypes.scala
new file mode 100644
index 0000000000..7235c0b9b9
--- /dev/null
+++
b/shims/spark34/src/main/scala/org/apache/spark/sql/classic/ClassicTypes.scala
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.classic
+
+import org.apache.spark.sql
+
+/**
+ * Prior to Spark 4.0, `ClassicSparkSession` refers to `sql.SparkSession`.
Since Spark 4.0, it
+ * refers to `sql.classic.SparkSession`.
+ */
+object ClassicTypes {
+
+ type ClassicSparkSession = sql.SparkSession
+}
diff --git
a/shims/spark35/src/main/scala/org/apache/spark/sql/classic/ClassicTypes.scala
b/shims/spark35/src/main/scala/org/apache/spark/sql/classic/ClassicTypes.scala
new file mode 100644
index 0000000000..7235c0b9b9
--- /dev/null
+++
b/shims/spark35/src/main/scala/org/apache/spark/sql/classic/ClassicTypes.scala
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.classic
+
+import org.apache.spark.sql
+
+/**
+ * Prior to Spark 4.0, `ClassicSparkSession` refers to `sql.SparkSession`.
Since Spark 4.0, it
+ * refers to `sql.classic.SparkSession`.
+ */
+object ClassicTypes {
+
+ type ClassicSparkSession = sql.SparkSession
+}
diff --git
a/shims/spark40/src/main/scala/org/apache/spark/sql/classic/ClassicTypes.scala
b/shims/spark40/src/main/scala/org/apache/spark/sql/classic/ClassicTypes.scala
new file mode 100644
index 0000000000..ea3fb8893f
--- /dev/null
+++
b/shims/spark40/src/main/scala/org/apache/spark/sql/classic/ClassicTypes.scala
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.classic
+
+import org.apache.spark.sql
+
+/**
+ * Prior to Spark 4.0, `ClassicSparkSession` refers to `sql.SparkSession`.
Since Spark 4.0, it
+ * refers to `sql.classic.SparkSession`.
+ */
+object ClassicTypes {
+
+ type ClassicSparkSession = sql.classic.SparkSession
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
