This is an automated email from the ASF dual-hosted git repository.
maxgekk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 6719168b6ec7 [SPARK-47327][SQL] Move sort keys concurrency test to
CollationFactorySuite
6719168b6ec7 is described below
commit 6719168b6ec72242e111bcb3aae75985d36fdad2
Author: Stefan Kandic <stefan.kan...@databricks.com>
AuthorDate: Sat Mar 16 09:24:22 2024 +0500
[SPARK-47327][SQL] Move sort keys concurrency test to CollationFactorySuite
### What changes were proposed in this pull request?
Move concurrency test to the `CollationFactorySuite`
### Why are the changes needed?
This is more appropriate location for the test as it directly uses the
`CollationFactory`.
Also, I just found out that `par` method is highly discouraged and that we
should use `ParSeq` instead.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
With existing UTs
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #45501 from stefankandic/moveTest.
Authored-by: Stefan Kandic <stefan.kan...@databricks.com>
Signed-off-by: Max Gekk <max.g...@gmail.com>
---
common/unsafe/pom.xml | 6 ++++++
.../apache/spark/unsafe/types/CollationFactorySuite.scala | 14 ++++++++++++++
.../test/scala/org/apache/spark/sql/CollationSuite.scala | 14 --------------
3 files changed, 20 insertions(+), 14 deletions(-)
diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml
index e9785ebb7ad4..13b45f55a4ad 100644
--- a/common/unsafe/pom.xml
+++ b/common/unsafe/pom.xml
@@ -47,6 +47,12 @@
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.scala-lang.modules</groupId>
+
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
+ <scope>test</scope>
+ </dependency>
+
<dependency>
<groupId>com.ibm.icu</groupId>
<artifactId>icu4j</artifactId>
diff --git
a/common/unsafe/src/test/scala/org/apache/spark/unsafe/types/CollationFactorySuite.scala
b/common/unsafe/src/test/scala/org/apache/spark/unsafe/types/CollationFactorySuite.scala
index f9927b94fd42..0a9ff7558e3a 100644
---
a/common/unsafe/src/test/scala/org/apache/spark/unsafe/types/CollationFactorySuite.scala
+++
b/common/unsafe/src/test/scala/org/apache/spark/unsafe/types/CollationFactorySuite.scala
@@ -17,6 +17,7 @@
package org.apache.spark.unsafe.types
+import scala.collection.parallel.immutable.ParSeq
import scala.jdk.CollectionConverters.MapHasAsScala
import org.apache.spark.SparkException
@@ -138,4 +139,17 @@ class CollationFactorySuite extends AnyFunSuite with
Matchers { // scalastyle:ig
assert(result == testCase.expectedResult)
})
}
+
+ test("test concurrently generating collation keys") {
+ // generating ICU sort keys is not thread-safe by default so this should
fail
+ // if we don't handle the concurrency properly on Collator level
+
+ (0 to 10).foreach(_ => {
+ val collator = fetchCollation("UNICODE").collator
+
+ ParSeq(0 to 100).foreach { _ =>
+ collator.getCollationKey("aaa")
+ }
+ })
+ }
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CollationSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/CollationSuite.scala
index bef7417be36c..aaf3e88c9bdb 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/CollationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CollationSuite.scala
@@ -18,7 +18,6 @@
package org.apache.spark.sql
import scala.collection.immutable.Seq
-import
scala.collection.parallel.CollectionConverters.ImmutableIterableIsParallelizable
import scala.jdk.CollectionConverters.MapHasAsJava
import org.apache.spark.SparkException
@@ -413,19 +412,6 @@ class CollationSuite extends DatasourceV2SQLBase with
AdaptiveSparkPlanHelper {
}
}
- test("test concurrently generating collation keys") {
- // generating ICU sort keys is not thread-safe by default so this should
fail
- // if we don't handle the concurrency properly on Collator level
-
- (0 to 10).foreach(_ => {
- val collator = CollationFactory.fetchCollation("UNICODE").collator
-
- (0 to 100).par.foreach { _ =>
- collator.getCollationKey("aaa")
- }
- })
- }
-
test("text writing to parquet with collation enclosed with backticks") {
withTempPath{ path =>
sql(s"select 'a' COLLATE `UNICODE`").write.parquet(path.getAbsolutePath)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
