This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 9169d93ed5b [SPARK-45700][SPARK-45702][SPARK-45703] Fix the compile
warning suppression rule added in SPARK-35496
9169d93ed5b is described below
commit 9169d93ed5b8cc53926a44ce497568d36cb275b0
Author: yangjie01 <yangji...@baidu.com>
AuthorDate: Tue Oct 31 08:53:54 2023 -0700
[SPARK-45700][SPARK-45702][SPARK-45703] Fix the compile warning suppression
rule added in SPARK-35496
### What changes were proposed in this pull request?
This PR fixed the issues corresponding to the compile suppression rules
added in SPARK-35496 for upgrading to Scala 2.13.7, and cleaned up these rules.
```
// SPARK-35496 Upgrade Scala to 2.13.7 and suppress:
// 1. `The outer reference in this type test cannot be checked at run time`
// 2. `the type test for pattern TypeA cannot be checked at runtime because
it
// has type parameters eliminated by erasure`
// 3. `abstract type TypeA in type pattern Seq[TypeA] (the underlying of
// Seq[TypeA]) is unchecked since it is eliminated by erasure`
// 4. `fruitless type test: a value of TypeA cannot also be a TypeB`
"-Wconf:cat=unchecked&msg=outer reference:s",
"-Wconf:cat=unchecked&msg=eliminated by erasure:s",
"-Wconf:msg=^(?=.*?a value of type)(?=.*?cannot also be).+$:s",
```
The specific fixes are as follows:
- Added the corresponding `outer reference` to fix issue 1
- Used fine-grained `unchecked` declarations to clean up issues 2 and 3
- Since the case corresponding to issue 4 no longer exists, the
corresponding suppression rule was directly deleted.
### Why are the changes needed?
Clean up the compile issue suppression rules added in SPARK-35496 for
upgrading to Scala 2.13.7.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Pass GitHub Actions
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #43582 from LuciferYang/SPARK-45702.
Lead-authored-by: yangjie01 <yangji...@baidu.com>
Co-authored-by: YangJie <yangji...@baidu.com>
Signed-off-by: Dongjoon Hyun <dh...@apple.com>
---
.../org/apache/spark/rdd/OrderedRDDFunctions.scala | 2 +-
.../spark/mllib/evaluation/RankingMetrics.scala | 6 ++++--
pom.xml | 12 -----------
project/SparkBuild.scala | 10 ---------
.../sql/catalyst/CatalystTypeConverters.scala | 2 +-
.../spark/sql/catalyst/plans/QueryPlan.scala | 8 ++++----
.../apache/spark/sql/catalyst/trees/TreeNode.scala | 2 +-
.../org/apache/spark/sql/SQLQueryTestSuite.scala | 24 +++++++++++-----------
.../thriftserver/ThriftServerQueryTestSuite.scala | 6 +++---
9 files changed, 26 insertions(+), 46 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/rdd/OrderedRDDFunctions.scala
b/core/src/main/scala/org/apache/spark/rdd/OrderedRDDFunctions.scala
index 2701f457ee8..965a57da8db 100644
--- a/core/src/main/scala/org/apache/spark/rdd/OrderedRDDFunctions.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/OrderedRDDFunctions.scala
@@ -97,7 +97,7 @@ class OrderedRDDFunctions[K : Ordering : ClassTag,
def inRange(k: K): Boolean = ordering.gteq(k, lower) && ordering.lteq(k,
upper)
val rddToFilter: RDD[P] = self.partitioner match {
- case Some(rp: RangePartitioner[K, V]) =>
+ case Some(rp: RangePartitioner[_, _]) =>
val partitionIndices = (rp.getPartition(lower),
rp.getPartition(upper)) match {
case (l, u) => Math.min(l, u) to Math.max(l, u)
}
diff --git
a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/RankingMetrics.scala
b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/RankingMetrics.scala
index 641f55bb05f..888b07dd4e6 100644
---
a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/RankingMetrics.scala
+++
b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/RankingMetrics.scala
@@ -44,8 +44,10 @@ class RankingMetrics[T: ClassTag] @Since("1.2.0")
(predictionAndLabels: RDD[_ <:
with Serializable {
private val rdd = predictionAndLabels.map {
- case (pred: Array[T], lab: Array[T]) => (pred, lab, Array.empty[Double])
- case (pred: Array[T], lab: Array[T], rel: Array[Double]) => (pred, lab,
rel)
+ case (pred: Array[T] @unchecked, lab: Array[T] @unchecked) =>
+ (pred, lab, Array.empty[Double])
+ case (pred: Array[T] @unchecked, lab: Array[T] @unchecked, rel:
Array[Double]) =>
+ (pred, lab, rel)
case _ => throw new IllegalArgumentException(s"Expected RDD of tuples or
triplets")
}
diff --git a/pom.xml b/pom.xml
index a76d8eaddb5..da8e713bd55 100644
--- a/pom.xml
+++ b/pom.xml
@@ -2981,18 +2981,6 @@
`procedure syntax is deprecated`
-->
<arg>-Wconf:cat=deprecation&msg=procedure syntax is
deprecated:e</arg>
- <!--
- SPARK-35496 Upgrade Scala to 2.13.7 and suppress:
- 1. `The outer reference in this type test cannot be checked at
run time`
- 2. `the type test for pattern TypeA cannot be checked at
runtime because it
- has type parameters eliminated by erasure`
- 3. `abstract type TypeA in type pattern Seq[TypeA] (the
underlying of
- Seq[TypeA]) is unchecked since it is eliminated by erasure`
- 4. `fruitless type test: a value of TypeA cannot also be a
TypeB`
- -->
- <arg>-Wconf:cat=unchecked&msg=outer reference:s</arg>
- <arg>-Wconf:cat=unchecked&msg=eliminated by erasure:s</arg>
- <arg>-Wconf:msg=^(?=.*?a value of type)(?=.*?cannot also
be).+$:s</arg>
<!--
SPARK-40497 Upgrade Scala to 2.13.11 and suppress `Implicit
definition should have explicit type`
-->
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 098a628ba1c..d9d4a836ab5 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -247,16 +247,6 @@ object SparkBuild extends PomBuild {
"-Wconf:cat=deprecation&msg=Auto-application to \\`\\(\\)\\` is
deprecated&site=org.apache.spark.streaming.kafka010.KafkaRDDSuite:s",
// SPARK-35574 Prevent the recurrence of compilation warnings related
to `procedure syntax is deprecated`
"-Wconf:cat=deprecation&msg=procedure syntax is deprecated:e",
- // SPARK-35496 Upgrade Scala to 2.13.7 and suppress:
- // 1. `The outer reference in this type test cannot be checked at run
time`
- // 2. `the type test for pattern TypeA cannot be checked at runtime
because it
- // has type parameters eliminated by erasure`
- // 3. `abstract type TypeA in type pattern Seq[TypeA] (the underlying
of
- // Seq[TypeA]) is unchecked since it is eliminated by erasure`
- // 4. `fruitless type test: a value of TypeA cannot also be a TypeB`
- "-Wconf:cat=unchecked&msg=outer reference:s",
- "-Wconf:cat=unchecked&msg=eliminated by erasure:s",
- "-Wconf:msg=^(?=.*?a value of type)(?=.*?cannot also be).+$:s",
// SPARK-40497 Upgrade Scala to 2.13.11 and suppress `Implicit
definition should have explicit type`
"-Wconf:msg=Implicit definition should have explicit type:s",
// SPARK-45331 Upgrade Scala to 2.13.12 and suppress "In Scala 2,
symbols inherited
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala
index fcb8a5d0545..fd063120c26 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala
@@ -102,7 +102,7 @@ object CatalystTypeConverters {
final def toCatalyst(@Nullable maybeScalaValue: Any): CatalystType = {
maybeScalaValue match {
case null | None => null.asInstanceOf[CatalystType]
- case opt: Some[ScalaInputType] => toCatalystImpl(opt.get)
+ case opt: Some[ScalaInputType @unchecked] => toCatalystImpl(opt.get)
case other => toCatalystImpl(other.asInstanceOf[ScalaInputType])
}
}
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
index c58a1c4c0f3..62ddf71ffcb 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
@@ -371,7 +371,7 @@ abstract class QueryPlan[PlanType <: QueryPlan[PlanType]]
transformExpressions {
case a: AttributeReference =>
updateAttr(a, attrMap)
- case pe: PlanExpression[PlanType] =>
+ case pe: PlanExpression[PlanType @unchecked] =>
pe.withNewPlan(updateOuterReferencesInSubquery(pe.plan, attrMap))
}.asInstanceOf[PlanType]
}
@@ -401,7 +401,7 @@ abstract class QueryPlan[PlanType <: QueryPlan[PlanType]]
currentFragment.transformExpressions {
case OuterReference(a: AttributeReference) =>
OuterReference(updateAttr(a, attrMap))
- case pe: PlanExpression[PlanType] =>
+ case pe: PlanExpression[PlanType @unchecked] =>
pe.withNewPlan(updateOuterReferencesInSubquery(pe.plan, attrMap))
}
}
@@ -490,7 +490,7 @@ abstract class QueryPlan[PlanType <: QueryPlan[PlanType]]
def transformUpWithSubqueries(f: PartialFunction[PlanType, PlanType]):
PlanType = {
transformUp { case plan =>
val transformed = plan transformExpressionsUp {
- case planExpression: PlanExpression[PlanType] =>
+ case planExpression: PlanExpression[PlanType @unchecked] =>
val newPlan = planExpression.plan.transformUpWithSubqueries(f)
planExpression.withNewPlan(newPlan)
}
@@ -524,7 +524,7 @@ abstract class QueryPlan[PlanType <: QueryPlan[PlanType]]
override def apply(plan: PlanType): PlanType = {
val transformed = f.applyOrElse[PlanType, PlanType](plan, identity)
transformed transformExpressionsDown {
- case planExpression: PlanExpression[PlanType] =>
+ case planExpression: PlanExpression[PlanType @unchecked] =>
val newPlan =
planExpression.plan.transformDownWithSubqueriesAndPruning(cond, ruleId)(f)
planExpression.withNewPlan(newPlan)
}
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
index f1a9888a2c4..c9425b24764 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
@@ -1090,7 +1090,7 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]]
// this child in all children.
case (name, value: TreeNode[_]) if containsChild(value) =>
name -> JInt(children.indexOf(value))
- case (name, value: Seq[BaseType]) if value.forall(containsChild) =>
+ case (name, value: Seq[BaseType @unchecked]) if
value.forall(containsChild) =>
name -> JArray(
value.map(v =>
JInt(children.indexOf(v.asInstanceOf[TreeNode[_]]))).toList
)
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
index d6dc5165fbb..eb5dbc6ef54 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
@@ -321,19 +321,19 @@ class SQLQueryTestSuite extends QueryTest with
SharedSparkSession with SQLHelper
// Create a test case to ignore this case.
ignore(testCase.name) { /* Do nothing */ }
} else testCase match {
- case udfTestCase: UDFTest
+ case udfTestCase: SQLQueryTestSuite#UDFTest
if udfTestCase.udf.isInstanceOf[TestPythonUDF] &&
!shouldTestPythonUDFs =>
ignore(s"${testCase.name} is skipped because " +
s"[$pythonExec] and/or pyspark were not available.") {
/* Do nothing */
}
- case udfTestCase: UDFTest
+ case udfTestCase: SQLQueryTestSuite#UDFTest
if udfTestCase.udf.isInstanceOf[TestScalarPandasUDF] &&
!shouldTestPandasUDFs =>
ignore(s"${testCase.name} is skipped because pyspark," +
s"pandas and/or pyarrow were not available in [$pythonExec].") {
/* Do nothing */
}
- case udfTestCase: UDFTest
+ case udfTestCase: SQLQueryTestSuite#UDFTest
if udfTestCase.udf.isInstanceOf[TestGroupedAggPandasUDF] &&
!shouldTestPandasUDFs =>
ignore(s"${testCase.name} is skipped because pyspark," +
@@ -503,15 +503,15 @@ class SQLQueryTestSuite extends QueryTest with
SharedSparkSession with SQLHelper
val localSparkSession = spark.newSession()
testCase match {
- case udfTestCase: UDFTest =>
+ case udfTestCase: SQLQueryTestSuite#UDFTest =>
registerTestUDF(udfTestCase.udf, localSparkSession)
- case udtfTestCase: UDTFSetTest =>
+ case udtfTestCase: SQLQueryTestSuite#UDTFSetTest =>
registerTestUDTFs(udtfTestCase.udtfSet, localSparkSession)
case _ =>
}
testCase match {
- case _: PgSQLTest =>
+ case _: SQLQueryTestSuite#PgSQLTest =>
// booleq/boolne used by boolean.sql
localSparkSession.udf.register("booleq", (b1: Boolean, b2: Boolean) =>
b1 == b2)
localSparkSession.udf.register("boolne", (b1: Boolean, b2: Boolean) =>
b1 != b2)
@@ -519,9 +519,9 @@ class SQLQueryTestSuite extends QueryTest with
SharedSparkSession with SQLHelper
localSparkSession.udf.register("vol", (s: String) => s)
localSparkSession.conf.set(SQLConf.ANSI_ENABLED.key, true)
localSparkSession.conf.set(SQLConf.LEGACY_INTERVAL_ENABLED.key, true)
- case _: AnsiTest =>
+ case _: SQLQueryTestSuite#AnsiTest =>
localSparkSession.conf.set(SQLConf.ANSI_ENABLED.key, true)
- case _: TimestampNTZTest =>
+ case _: SQLQueryTestSuite#TimestampNTZTest =>
localSparkSession.conf.set(SQLConf.TIMESTAMP_TYPE.key,
TimestampTypes.TIMESTAMP_NTZ.toString)
case _ =>
@@ -581,19 +581,19 @@ class SQLQueryTestSuite extends QueryTest with
SharedSparkSession with SQLHelper
// See also SPARK-29127. It is difficult to see the version information in
the failed test
// cases so the version information related to Python was also added.
val clue = testCase match {
- case udfTestCase: UDFTest
+ case udfTestCase: SQLQueryTestSuite#UDFTest
if udfTestCase.udf.isInstanceOf[TestPythonUDF] &&
shouldTestPythonUDFs =>
s"${testCase.name}${System.lineSeparator()}Python:
$pythonVer${System.lineSeparator()}"
- case udfTestCase: UDFTest
+ case udfTestCase: SQLQueryTestSuite#UDFTest
if udfTestCase.udf.isInstanceOf[TestScalarPandasUDF] &&
shouldTestPandasUDFs =>
s"${testCase.name}${System.lineSeparator()}" +
s"Python: $pythonVer Pandas: $pandasVer PyArrow:
$pyarrowVer${System.lineSeparator()}"
- case udfTestCase: UDFTest
+ case udfTestCase: SQLQueryTestSuite#UDFTest
if udfTestCase.udf.isInstanceOf[TestGroupedAggPandasUDF] &&
shouldTestPandasUDFs =>
s"${testCase.name}${System.lineSeparator()}" +
s"Python: $pythonVer Pandas: $pandasVer PyArrow:
$pyarrowVer${System.lineSeparator()}"
- case udtfTestCase: UDTFSetTest
+ case udtfTestCase: SQLQueryTestSuite#UDTFSetTest
if udtfTestCase.udtfSet.udtfs.forall(_.isInstanceOf[TestPythonUDTF])
&&
shouldTestPythonUDFs =>
s"${testCase.name}${System.lineSeparator()}Python:
$pythonVer${System.lineSeparator()}"
diff --git
a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala
b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala
index adbc42ab245..ee336b13b0b 100644
---
a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala
+++
b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala
@@ -116,12 +116,12 @@ class ThriftServerQueryTestSuite extends
SQLQueryTestSuite with SharedThriftServ
}
testCase match {
- case _: PgSQLTest =>
+ case _: SQLQueryTestSuite#PgSQLTest =>
statement.execute(s"SET ${SQLConf.ANSI_ENABLED.key} = true")
statement.execute(s"SET ${SQLConf.LEGACY_INTERVAL_ENABLED.key} =
true")
- case _: AnsiTest =>
+ case _: SQLQueryTestSuite#AnsiTest =>
statement.execute(s"SET ${SQLConf.ANSI_ENABLED.key} = true")
- case _: TimestampNTZTest =>
+ case _: SQLQueryTestSuite#TimestampNTZTest =>
statement.execute(s"SET ${SQLConf.TIMESTAMP_TYPE.key} = " +
s"${TimestampTypes.TIMESTAMP_NTZ.toString}")
case _ =>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
