This is an automated email from the ASF dual-hosted git repository. yangjie01 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 0a45a8662db [SPARK-30848][SQL] Remove productHash from TreeNode 0a45a8662db is described below commit 0a45a8662dbdeb644ea3950cdaa6e3ca328cd6fc Author: panbingkun <pbk1...@gmail.com> AuthorDate: Sat Oct 21 00:07:01 2023 +0800 [SPARK-30848][SQL] Remove productHash from TreeNode ### What changes were proposed in this pull request? The pr aims to replace `TreeNode.productHash` with `MurmurHash3.productHash`. ### Why are the changes needed? 1.Spark 4.0 no longer supports Scala 2.12. 2.Using MurmurHash3's class method `productHash` to reduce code redundancy. https://github.com/scala/scala/blob/v2.13.11/src/library/scala/util/hashing/MurmurHash3.scala#L343 https://github.com/scala/scala/blob/v2.13.11/src/library/scala/util/hashing/MurmurHash3.scala#L64-L81 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? 1.Pass GA. 2.Manually test: ``` (base) panbingkun:~/Developer/spark/spark-community$./build/sbt "catalyst/testOnly org.apache.spark.sql.catalyst.expressions.CanonicalizeSuite -- -t \"SPARK-30847: Take productPrefix into account in MurmurHash3.productHash\"" [info] CanonicalizeSuite: [info] - SPARK-30847: Take productPrefix into account in MurmurHash3.productHash (249 milliseconds) [info] Run completed in 1 second, 135 milliseconds. [info] Total number of tests run: 1 [info] Suites: completed 1, aborted 0 [info] Tests: succeeded 1, failed 0, canceled 0, ignored 0, pending 0 [info] All tests passed. [success] Total time: 230 s (03:50), completed Sep 28, 2023, 12:54:06 PM ``` ### Was this patch authored or co-authored using generative AI tooling? No. Closes #43161 from panbingkun/SPARK-45366. Authored-by: panbingkun <pbk1...@gmail.com> Signed-off-by: yangjie01 <yangji...@baidu.com> --- .../apache/spark/sql/catalyst/trees/TreeNode.scala | 26 ++-------------------- 1 file changed, 2 insertions(+), 24 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala index cc470d0de6f..e7be8a7e29b 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala @@ -22,6 +22,7 @@ import java.util.UUID import scala.collection.{mutable, Map} import scala.jdk.CollectionConverters._ import scala.reflect.ClassTag +import scala.util.hashing.MurmurHash3 import org.apache.commons.lang3.ClassUtils import org.json4s.JsonAST._ @@ -175,30 +176,7 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] lazy val height: Int = children.map(_.height).reduceOption(_ max _).getOrElse(0) + 1 - // Copied from Scala 2.13.1 - // github.com/scala/scala/blob/v2.13.1/src/library/scala/util/hashing/MurmurHash3.scala#L56-L73 - // to prevent the issue https://github.com/scala/bug/issues/10495 - // TODO(SPARK-30848): Remove this once we drop Scala 2.12. - private final def productHash(x: Product, seed: Int, ignorePrefix: Boolean = false): Int = { - val arr = x.productArity - // Case objects have the hashCode inlined directly into the - // synthetic hashCode method, but this method should still give - // a correct result if passed a case object. - if (arr == 0) { - x.productPrefix.hashCode - } else { - var h = seed - if (!ignorePrefix) h = scala.util.hashing.MurmurHash3.mix(h, x.productPrefix.hashCode) - var i = 0 - while (i < arr) { - h = scala.util.hashing.MurmurHash3.mix(h, x.productElement(i).##) - i += 1 - } - scala.util.hashing.MurmurHash3.finalizeHash(h, arr) - } - } - - private lazy val _hashCode: Int = productHash(this, scala.util.hashing.MurmurHash3.productSeed) + private lazy val _hashCode: Int = MurmurHash3.productHash(this) override def hashCode(): Int = _hashCode /** --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org