This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new a4e268ca test: Add tests for Scalar and Inverval values for UnaryMinus
(#538)
a4e268ca is described below
commit a4e268ca7771ee43b0b312d33afd8e9d9f4e37e9
Author: Vipul Vaibhaw <[email protected]>
AuthorDate: Tue Jun 11 22:16:36 2024 +0530
test: Add tests for Scalar and Inverval values for UnaryMinus (#538)
* adding scalar tests
* refactor and test for interval
* ci checks fixed
* running operator checks when no error
* removing redundant sqlconf
* fix ci errorsg
* moving interval test to array section
* ci fixes
---
.../org/apache/comet/CometExpressionSuite.scala | 102 ++++++++++-----------
1 file changed, 47 insertions(+), 55 deletions(-)
diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala
b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala
index a2b6edd0..9a1851a5 100644
--- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala
+++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala
@@ -19,6 +19,8 @@
package org.apache.comet
+import java.time.{Duration, Period}
+
import scala.reflect.ClassTag
import scala.reflect.runtime.universe.TypeTag
@@ -1635,7 +1637,7 @@ class CometExpressionSuite extends CometTestBase with
AdaptiveSparkPlanHelper {
case (Some(sparkException), Some(cometException)) =>
assert(sparkException.getMessage.contains(dtype + " overflow"))
assert(cometException.getMessage.contains(dtype + " overflow"))
- case (None, None) => assert(true) // got same outputs
+ case (None, None) => checkSparkAnswerAndOperator(sql(query))
case (None, Some(ex)) =>
fail("Comet threw an exception but Spark did not " + ex.getMessage)
case (Some(_), None) =>
@@ -1656,66 +1658,56 @@ class CometExpressionSuite extends CometTestBase with
AdaptiveSparkPlanHelper {
withTempDir { dir =>
// Array values test
- val arrayPath = new Path(dir.toURI.toString,
"array_test.parquet").toString
- Seq(Int.MaxValue,
Int.MinValue).toDF("a").write.mode("overwrite").parquet(arrayPath)
- val arrayQuery = "select a, -a from t"
- runArrayTest(arrayQuery, "integer", arrayPath)
-
- // long values test
- val longArrayPath = new Path(dir.toURI.toString,
"long_array_test.parquet").toString
- Seq(Long.MaxValue, Long.MinValue)
- .toDF("a")
- .write
- .mode("overwrite")
- .parquet(longArrayPath)
- val longArrayQuery = "select a, -a from t"
- runArrayTest(longArrayQuery, "long", longArrayPath)
-
- // short values test
- val shortArrayPath = new Path(dir.toURI.toString,
"short_array_test.parquet").toString
- Seq(Short.MaxValue, Short.MinValue)
- .toDF("a")
- .write
- .mode("overwrite")
- .parquet(shortArrayPath)
- val shortArrayQuery = "select a, -a from t"
- runArrayTest(shortArrayQuery, "", shortArrayPath)
-
- // byte values test
- val byteArrayPath = new Path(dir.toURI.toString,
"byte_array_test.parquet").toString
- Seq(Byte.MaxValue, Byte.MinValue)
- .toDF("a")
- .write
- .mode("overwrite")
- .parquet(byteArrayPath)
- val byteArrayQuery = "select a, -a from t"
- runArrayTest(byteArrayQuery, "", byteArrayPath)
-
- // interval values test
- withTable("t_interval") {
- spark.sql("CREATE TABLE t_interval(a STRING) USING PARQUET")
- spark.sql("INSERT INTO t_interval VALUES ('INTERVAL 10000000000
YEAR')")
- withAnsiMode(enabled = true) {
- spark
- .sql("SELECT CAST(a AS INTERVAL) AS a FROM t_interval")
- .createOrReplaceTempView("t_interval_casted")
- checkOverflow("SELECT a, -a FROM t_interval_casted", "interval")
- }
+ val dataTypes = Seq(
+ ("array_test.parquet", Seq(Int.MaxValue, Int.MinValue).toDF("a"),
"integer"),
+ ("long_array_test.parquet", Seq(Long.MaxValue,
Long.MinValue).toDF("a"), "long"),
+ ("short_array_test.parquet", Seq(Short.MaxValue,
Short.MinValue).toDF("a"), ""),
+ ("byte_array_test.parquet", Seq(Byte.MaxValue,
Byte.MinValue).toDF("a"), ""))
+
+ dataTypes.foreach { case (fileName, df, dtype) =>
+ val path = new Path(dir.toURI.toString, fileName).toString
+ df.write.mode("overwrite").parquet(path)
+ val query = "select a, -a from t"
+ runArrayTest(query, dtype, path)
}
- withTable("t") {
- sql("create table t(a int) using parquet")
- sql("insert into t values (-2147483648)")
+ withParquetTable((0 until 5).map(i => (i % 5, i % 3)), "tbl") {
withAnsiMode(enabled = true) {
- checkOverflow("select a, -a from t", "integer")
+ // interval test without cast
+ val longDf = Seq(Long.MaxValue, Long.MaxValue, 2)
+ val yearMonthDf = Seq(Int.MaxValue, Int.MaxValue, 2)
+ .map(Period.ofMonths)
+ val dayTimeDf = Seq(106751991L, 106751991L, 2L)
+ .map(Duration.ofDays)
+ Seq(longDf, yearMonthDf, dayTimeDf).foreach { _ =>
+ checkOverflow("select -(_1) FROM tbl", "")
+ }
}
}
- withTable("t_float") {
- sql("create table t_float(a float) using parquet")
- sql("insert into t_float values (3.4128235E38)")
- withAnsiMode(enabled = true) {
- checkOverflow("select a, -a from t_float", "float")
+ // scalar tests
+ withParquetTable((0 until 5).map(i => (i % 5, i % 3)), "tbl") {
+ withSQLConf(
+ "spark.sql.optimizer.excludedRules" ->
"org.apache.spark.sql.catalyst.optimizer.ConstantFolding",
+ SQLConf.ANSI_ENABLED.key -> "true",
+ CometConf.COMET_ANSI_MODE_ENABLED.key -> "true",
+ CometConf.COMET_ENABLED.key -> "true",
+ CometConf.COMET_EXEC_ENABLED.key -> "true") {
+ for (n <- Seq("2147483647", "-2147483648")) {
+ checkOverflow(s"select -(cast(${n} as int)) FROM tbl", "integer")
+ }
+ for (n <- Seq("32767", "-32768")) {
+ checkOverflow(s"select -(cast(${n} as short)) FROM tbl", "")
+ }
+ for (n <- Seq("127", "-128")) {
+ checkOverflow(s"select -(cast(${n} as byte)) FROM tbl", "")
+ }
+ for (n <- Seq("9223372036854775807", "-9223372036854775808")) {
+ checkOverflow(s"select -(cast(${n} as long)) FROM tbl", "long")
+ }
+ for (n <- Seq("3.4028235E38", "-3.4028235E38")) {
+ checkOverflow(s"select -(cast(${n} as float)) FROM tbl", "float")
+ }
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
