JoshRosen commented on code in PR #57710:
URL: https://github.com/apache/spark/pull/57710#discussion_r3705669047
##########
common/utils/src/main/scala/org/apache/spark/util/ClosureCleaner.scala:
##########
@@ -34,6 +35,39 @@ import org.apache.spark.internal.Logging
* A cleaner that renders closures serializable if they can be done so safely.
*/
private[spark] object ClosureCleaner extends Logging {
+ /**
+ * Per-class memo of which closure methods contain a non-local return, i.e.
allocate a
+ * `scala/runtime/NonLocalReturnControl`. The verdict is a pure function of
the class's
+ * immutable bytecode, so one ASM parse per class answers for every
`clean()` call.
+ */
+ private val methodsWithNonLocalReturn = new
ClassValue[immutable.Set[String]] {
+ override def computeValue(cls: Class[_]): immutable.Set[String] = {
+ val collector = new ReturnStatementCollector
+ val reader = getClassReader(cls)
+ if (reader != null) {
+ reader.accept(collector, 0)
+ } else {
+ logDebug(s"Cannot get class bytes for ${cls.getName}; skipping
return-statement check")
+ }
+ collector.found.toSet
+ }
+ }
+
+ /** Whether `implMethodName` (any closure method, if `None`) of `cls` has a
non-local return. */
+ private[util] def hasReturnStatement(cls: Class[_], implMethodName:
Option[String]): Boolean = {
+ val found = methodsWithNonLocalReturn.get(cls)
+ implMethodName match {
+ case None => found.nonEmpty
+ case Some(target) =>
+ // A method with suffix "$adapted" will be generated in cases like
+ // { _:Int => return; Seq()} but not { _:Int => return; true}
+ // closure passed is $anonfun$t$1$adapted while actual code resides in
$anonfun$s$1
+ // the class file only contains $anonfun$s$1, so we remove the suffix,
see
Review Comment:
Good catch; I corrected the comment in
https://github.com/apache/spark/pull/57710/changes/2fcbbb438dcee7a54c639c916fb048359923fd58
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
