This is an automated email from the ASF dual-hosted git repository. haejoon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new c9868b5dadc2 [SPARK-48510][CONNECT][FOLLOW-UP] Fix for UDAF `toColumn` API when running tests in Maven c9868b5dadc2 is described below commit c9868b5dadc2cfb20a2eadb180cd65eae45bbb2c Author: Paddy Xu <xupa...@gmail.com> AuthorDate: Wed Jul 17 10:38:32 2024 +0900 [SPARK-48510][CONNECT][FOLLOW-UP] Fix for UDAF `toColumn` API when running tests in Maven ### What changes were proposed in this pull request? This PR fixes an issue where the TypeTag look up during `udaf.toColumn` failed in Maven test env with the following error: > java.lang.IllegalArgumentException: Type tag defined in [JavaMirror with jdk.internal.loader.ClassLoaders$AppClassLoader1dbd16a6 of type class jdk.internal.loader.ClassLoaders$AppClassLoader with classpath [<unknown>] and parent being jdk.internal.loader.ClassLoaders$PlatformClassLoader6bd61f98 of type class jdk.internal.loader.ClassLoaders$PlatformClassLoader with classpath [<unknown>] and parent being primordial classloader with boot classpath [<unknown>]] cannot be migrated to [...] The problem is caused by Maven adding a `URLClassLoader` on top of the original `AppClassLoader` (see the underlined texts in the above error message). This PR changes the mirror-matching logic from `eq` to `hasCommonAncestors`. ### Why are the changes needed? Previous logic fails in tests env. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Existing tests. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #47368 from xupefei/udaf-tocolumn-fixup. Authored-by: Paddy Xu <xupa...@gmail.com> Signed-off-by: Haejoon Lee <haejoon....@databricks.com> --- .../scala/org/apache/spark/sql/expressions/Aggregator.scala | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/expressions/Aggregator.scala b/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/expressions/Aggregator.scala index 3dabcdef1567..8ef7ccf22586 100644 --- a/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/expressions/Aggregator.scala +++ b/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/expressions/Aggregator.scala @@ -132,15 +132,24 @@ abstract class Aggregator[-IN, BUF, OUT] extends Serializable { val inType = typeArgs.head import scala.reflect.api._ + def areCompatibleMirrors(one: Mirror[_], another: Mirror[_]): Boolean = { + (one, another) match { + case (a: JavaMirror, b: JavaMirror) => + Iterator.iterate(b.classLoader)(_.getParent).contains(a.classLoader) || + Iterator.iterate(a.classLoader)(_.getParent).contains(b.classLoader) + case _ => one == another + } + } + TypeTag( mirror, new TypeCreator { def apply[U <: Universe with Singleton](m: Mirror[U]): U#Type = - if (m eq mirror) { + if (areCompatibleMirrors(m, mirror)) { inType.asInstanceOf[U#Type] } else { throw new IllegalArgumentException( - s"Type tag defined in $mirror cannot be migrated to other mirrors.") + s"Type tag defined in [$mirror] cannot be migrated to another mirror [$m].") } }) } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org