This is an automated email from the ASF dual-hosted git repository. maxgekk pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 6719168b6ec7 [SPARK-47327][SQL] Move sort keys concurrency test to CollationFactorySuite 6719168b6ec7 is described below commit 6719168b6ec72242e111bcb3aae75985d36fdad2 Author: Stefan Kandic <stefan.kan...@databricks.com> AuthorDate: Sat Mar 16 09:24:22 2024 +0500 [SPARK-47327][SQL] Move sort keys concurrency test to CollationFactorySuite ### What changes were proposed in this pull request? Move concurrency test to the `CollationFactorySuite` ### Why are the changes needed? This is more appropriate location for the test as it directly uses the `CollationFactory`. Also, I just found out that `par` method is highly discouraged and that we should use `ParSeq` instead. ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? With existing UTs ### Was this patch authored or co-authored using generative AI tooling? No Closes #45501 from stefankandic/moveTest. Authored-by: Stefan Kandic <stefan.kan...@databricks.com> Signed-off-by: Max Gekk <max.g...@gmail.com> --- common/unsafe/pom.xml | 6 ++++++ .../apache/spark/unsafe/types/CollationFactorySuite.scala | 14 ++++++++++++++ .../test/scala/org/apache/spark/sql/CollationSuite.scala | 14 -------------- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml index e9785ebb7ad4..13b45f55a4ad 100644 --- a/common/unsafe/pom.xml +++ b/common/unsafe/pom.xml @@ -47,6 +47,12 @@ <version>${project.version}</version> </dependency> + <dependency> + <groupId>org.scala-lang.modules</groupId> + <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId> + <scope>test</scope> + </dependency> + <dependency> <groupId>com.ibm.icu</groupId> <artifactId>icu4j</artifactId> diff --git a/common/unsafe/src/test/scala/org/apache/spark/unsafe/types/CollationFactorySuite.scala b/common/unsafe/src/test/scala/org/apache/spark/unsafe/types/CollationFactorySuite.scala index f9927b94fd42..0a9ff7558e3a 100644 --- a/common/unsafe/src/test/scala/org/apache/spark/unsafe/types/CollationFactorySuite.scala +++ b/common/unsafe/src/test/scala/org/apache/spark/unsafe/types/CollationFactorySuite.scala @@ -17,6 +17,7 @@ package org.apache.spark.unsafe.types +import scala.collection.parallel.immutable.ParSeq import scala.jdk.CollectionConverters.MapHasAsScala import org.apache.spark.SparkException @@ -138,4 +139,17 @@ class CollationFactorySuite extends AnyFunSuite with Matchers { // scalastyle:ig assert(result == testCase.expectedResult) }) } + + test("test concurrently generating collation keys") { + // generating ICU sort keys is not thread-safe by default so this should fail + // if we don't handle the concurrency properly on Collator level + + (0 to 10).foreach(_ => { + val collator = fetchCollation("UNICODE").collator + + ParSeq(0 to 100).foreach { _ => + collator.getCollationKey("aaa") + } + }) + } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CollationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CollationSuite.scala index bef7417be36c..aaf3e88c9bdb 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/CollationSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/CollationSuite.scala @@ -18,7 +18,6 @@ package org.apache.spark.sql import scala.collection.immutable.Seq -import scala.collection.parallel.CollectionConverters.ImmutableIterableIsParallelizable import scala.jdk.CollectionConverters.MapHasAsJava import org.apache.spark.SparkException @@ -413,19 +412,6 @@ class CollationSuite extends DatasourceV2SQLBase with AdaptiveSparkPlanHelper { } } - test("test concurrently generating collation keys") { - // generating ICU sort keys is not thread-safe by default so this should fail - // if we don't handle the concurrency properly on Collator level - - (0 to 10).foreach(_ => { - val collator = CollationFactory.fetchCollation("UNICODE").collator - - (0 to 100).par.foreach { _ => - collator.getCollationKey("aaa") - } - }) - } - test("text writing to parquet with collation enclosed with backticks") { withTempPath{ path => sql(s"select 'a' COLLATE `UNICODE`").write.parquet(path.getAbsolutePath) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org